add support for exclusion rules in dockerignore

Signed-off-by: Dave Goodchild <buddhamagnet@gmail.com>

archive/archive.go

@@ -391,6 +391,13 @@ func Tar(path string, compression Compression) (io.ReadCloser, error) {
 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
+
+	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)
+
+	if err != nil {
+		return nil, err
+	}
+
 	pipeReader, pipeWriter := io.Pipe()
 
 	compressWriter, err := CompressStream(pipeWriter, options.Compression)
@@ -441,7 +448,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 				// is asking for that file no matter what - which is true
 				// for some files, like .dockerignore and Dockerfile (sometimes)
 				if include != relFilePath {
-					skip, err = fileutils.Matches(relFilePath, options.ExcludePatterns)
+					skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs)
 					if err != nil {
 						logrus.Debugf("Error matching %s", relFilePath, err)
 						return err
@@ -449,7 +456,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 				}
 
 				if skip {
-					if f.IsDir() {
+					if !exceptions && f.IsDir() {
 						return filepath.SkipDir
 					}
 					return nil

fileutils/fileutils.go

@@ -1,33 +1,120 @@
 package fileutils
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
 )
 
-// Matches returns true if relFilePath matches any of the patterns
-func Matches(relFilePath string, patterns []string) (bool, error) {
-	for _, exclude := range patterns {
-		matched, err := filepath.Match(exclude, relFilePath)
+func Exclusion(pattern string) bool {
+	return pattern[0] == '!'
+}
+
+func Empty(pattern string) bool {
+	return pattern == ""
+}
+
+// Cleanpatterns takes a slice of patterns returns a new
+// slice of patterns cleaned with filepath.Clean, stripped
+// of any empty patterns and lets the caller know whether the
+// slice contains any exception patterns (prefixed with !).
+func CleanPatterns(patterns []string) ([]string, [][]string, bool, error) {
+	// Loop over exclusion patterns and:
+	// 1. Clean them up.
+	// 2. Indicate whether we are dealing with any exception rules.
+	// 3. Error if we see a single exclusion marker on it's own (!).
+	cleanedPatterns := []string{}
+	patternDirs := [][]string{}
+	exceptions := false
+	for _, pattern := range patterns {
+		// Eliminate leading and trailing whitespace.
+		pattern = strings.TrimSpace(pattern)
+		if Empty(pattern) {
+			continue
+		}
+		if Exclusion(pattern) {
+			if len(pattern) == 1 {
+				logrus.Errorf("Illegal exclusion pattern: %s", pattern)
+				return nil, nil, false, errors.New("Illegal exclusion pattern: !")
+			}
+			exceptions = true
+		}
+		pattern = filepath.Clean(pattern)
+		cleanedPatterns = append(cleanedPatterns, pattern)
+		if Exclusion(pattern) {
+			pattern = pattern[1:]
+		}
+		patternDirs = append(patternDirs, strings.Split(pattern, "/"))
+	}
+
+	return cleanedPatterns, patternDirs, exceptions, nil
+}
+
+// Matches returns true if file matches any of the patterns
+// and isn't excluded by any of the subsequent patterns.
+func Matches(file string, patterns []string) (bool, error) {
+	file = filepath.Clean(file)
+
+	if file == "." {
+		// Don't let them exclude everything, kind of silly.
+		return false, nil
+	}
+
+	patterns, patDirs, _, err := CleanPatterns(patterns)
+	if err != nil {
+		return false, err
+	}
+
+	return OptimizedMatches(file, patterns, patDirs)
+}
+
+// Matches is basically the same as fileutils.Matches() but optimized for archive.go.
+// It will assume that the inputs have been preprocessed and therefore the function
+// doen't need to do as much error checking and clean-up. This was done to avoid
+// repeating these steps on each file being checked during the archive process.
+// The more generic fileutils.Matches() can't make these assumptions.
+func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool, error) {
+	matched := false
+	parentPath := filepath.Dir(file)
+	parentPathDirs := strings.Split(parentPath, "/")
+
+	for i, pattern := range patterns {
+		negative := false
+
+		if Exclusion(pattern) {
+			negative = true
+			pattern = pattern[1:]
+		}
+
+		match, err := filepath.Match(pattern, file)
 		if err != nil {
-			logrus.Errorf("Error matching: %s (pattern: %s)", relFilePath, exclude)
+			logrus.Errorf("Error matching: %s (pattern: %s)", file, pattern)
 			return false, err
 		}
-		if matched {
-			if filepath.Clean(relFilePath) == "." {
-				logrus.Errorf("Can't exclude whole path, excluding pattern: %s", exclude)
-				continue
+
+		if !match && parentPath != "." {
+			// Check to see if the pattern matches one of our parent dirs.
+			if len(patDirs[i]) <= len(parentPathDirs) {
+				match, _ = filepath.Match(strings.Join(patDirs[i], "/"),
+					strings.Join(parentPathDirs[:len(patDirs[i])], "/"))
 			}
-			logrus.Debugf("Skipping excluded path: %s", relFilePath)
-			return true, nil
+		}
+
+		if match {
+			matched = !negative
 		}
 	}
-	return false, nil
+
+	if matched {
+		logrus.Debugf("Skipping excluded path: %s", file)
+	}
+	return matched, nil
 }
 
 func CopyFile(src, dst string) (int64, error) {

fileutils/fileutils_test.go

@@ -79,3 +79,142 @@ func TestReadSymlinkedDirectoryToFile(t *testing.T) {
 		t.Errorf("failed to remove symlink: %s", err)
 	}
 }
+
+func TestWildcardMatches(t *testing.T) {
+	match, _ := Matches("fileutils.go", []string{"*"})
+	if match != true {
+		t.Errorf("failed to get a wildcard match, got %v", match)
+	}
+}
+
+// A simple pattern match should return true.
+func TestPatternMatches(t *testing.T) {
+	match, _ := Matches("fileutils.go", []string{"*.go"})
+	if match != true {
+		t.Errorf("failed to get a match, got %v", match)
+	}
+}
+
+// An exclusion followed by an inclusion should return true.
+func TestExclusionPatternMatchesPatternBefore(t *testing.T) {
+	match, _ := Matches("fileutils.go", []string{"!fileutils.go", "*.go"})
+	if match != true {
+		t.Errorf("failed to get true match on exclusion pattern, got %v", match)
+	}
+}
+
+// A folder pattern followed by an exception should return false.
+func TestPatternMatchesFolderExclusions(t *testing.T) {
+	match, _ := Matches("docs/README.md", []string{"docs", "!docs/README.md"})
+	if match != false {
+		t.Errorf("failed to get a false match on exclusion pattern, got %v", match)
+	}
+}
+
+// A folder pattern followed by an exception should return false.
+func TestPatternMatchesFolderWithSlashExclusions(t *testing.T) {
+	match, _ := Matches("docs/README.md", []string{"docs/", "!docs/README.md"})
+	if match != false {
+		t.Errorf("failed to get a false match on exclusion pattern, got %v", match)
+	}
+}
+
+// A folder pattern followed by an exception should return false.
+func TestPatternMatchesFolderWildcardExclusions(t *testing.T) {
+	match, _ := Matches("docs/README.md", []string{"docs/*", "!docs/README.md"})
+	if match != false {
+		t.Errorf("failed to get a false match on exclusion pattern, got %v", match)
+	}
+}
+
+// A pattern followed by an exclusion should return false.
+func TestExclusionPatternMatchesPatternAfter(t *testing.T) {
+	match, _ := Matches("fileutils.go", []string{"*.go", "!fileutils.go"})
+	if match != false {
+		t.Errorf("failed to get false match on exclusion pattern, got %v", match)
+	}
+}
+
+// A filename evaluating to . should return false.
+func TestExclusionPatternMatchesWholeDirectory(t *testing.T) {
+	match, _ := Matches(".", []string{"*.go"})
+	if match != false {
+		t.Errorf("failed to get false match on ., got %v", match)
+	}
+}
+
+// A single ! pattern should return an error.
+func TestSingleExclamationError(t *testing.T) {
+	_, err := Matches("fileutils.go", []string{"!"})
+	if err == nil {
+		t.Errorf("failed to get an error for a single exclamation point, got %v", err)
+	}
+}
+
+// A string preceded with a ! should return true from Exclusion.
+func TestExclusion(t *testing.T) {
+	exclusion := Exclusion("!")
+	if !exclusion {
+		t.Errorf("failed to get true for a single !, got %v", exclusion)
+	}
+}
+
+// An empty string should return true from Empty.
+func TestEmpty(t *testing.T) {
+	empty := Empty("")
+	if !empty {
+		t.Errorf("failed to get true for an empty string, got %v", empty)
+	}
+}
+
+func TestCleanPatterns(t *testing.T) {
+	cleaned, _, _, _ := CleanPatterns([]string{"docs", "config"})
+	if len(cleaned) != 2 {
+		t.Errorf("expected 2 element slice, got %v", len(cleaned))
+	}
+}
+
+func TestCleanPatternsStripEmptyPatterns(t *testing.T) {
+	cleaned, _, _, _ := CleanPatterns([]string{"docs", "config", ""})
+	if len(cleaned) != 2 {
+		t.Errorf("expected 2 element slice, got %v", len(cleaned))
+	}
+}
+
+func TestCleanPatternsExceptionFlag(t *testing.T) {
+	_, _, exceptions, _ := CleanPatterns([]string{"docs", "!docs/README.md"})
+	if !exceptions {
+		t.Errorf("expected exceptions to be true, got %v", exceptions)
+	}
+}
+
+func TestCleanPatternsLeadingSpaceTrimmed(t *testing.T) {
+	_, _, exceptions, _ := CleanPatterns([]string{"docs", "  !docs/README.md"})
+	if !exceptions {
+		t.Errorf("expected exceptions to be true, got %v", exceptions)
+	}
+}
+
+func TestCleanPatternsTrailingSpaceTrimmed(t *testing.T) {
+	_, _, exceptions, _ := CleanPatterns([]string{"docs", "!docs/README.md  "})
+	if !exceptions {
+		t.Errorf("expected exceptions to be true, got %v", exceptions)
+	}
+}
+
+func TestCleanPatternsErrorSingleException(t *testing.T) {
+	_, _, _, err := CleanPatterns([]string{"!"})
+	if err == nil {
+		t.Errorf("expected error on single exclamation point, got %v", err)
+	}
+}
+
+func TestCleanPatternsFolderSplit(t *testing.T) {
+	_, dirs, _, _ := CleanPatterns([]string{"docs/config/CONFIG.md"})
+	if dirs[0][0] != "docs" {
+		t.Errorf("expected first element in dirs slice to be docs, got %v", dirs[0][1])
+	}
+	if dirs[0][1] != "config" {
+		t.Errorf("expected first element in dirs slice to be config, got %v", dirs[0][1])
+	}
+}