diff --git a/docs/sources/articles/dockerfile_best-practices.md b/docs/sources/articles/dockerfile_best-practices.md index 425eb86583..dfb265c68b 100644 --- a/docs/sources/articles/dockerfile_best-practices.md +++ b/docs/sources/articles/dockerfile_best-practices.md @@ -32,13 +32,14 @@ ephemeral as possible. By “ephemeral,” we mean that it can be stopped and destroyed and a new one built and put in place with an absolute minimum of set-up and configuration. -### Use [a .dockerignore file](https://docs.docker.com/reference/builder/#the-dockerignore-file) +### Use a .dockerignore file -For faster uploading and efficiency during `docker build`, you should use -a `.dockerignore` file to exclude files or directories from the build -context and final image. For example, unless`.git` is needed by your build -process or scripts, you should add it to `.dockerignore`, which can save many -megabytes worth of upload time. +In most cases, it's best to put each Dockerfile in an empty directory. Then, +add to that directory only the files needed for building the Dockerfile. To +increase the build's performance, you can exclude files and directories by +adding a `.dockerignore` file to that directory as well. This file supports +exclusion patterns similar to `.gitignore` files. For information on creating one, +see the [.dockerignore file](../../reference/builder/#dockerignore-file). ### Avoid installing unnecessary packages diff --git a/docs/sources/reference/builder.md b/docs/sources/reference/builder.md index c5392f8474..7dbe549237 100644 --- a/docs/sources/reference/builder.md +++ b/docs/sources/reference/builder.md @@ -41,10 +41,11 @@ whole context must be transferred to the daemon. The Docker CLI reports > repository, the entire contents of your hard drive will get sent to the daemon (and > thus to the machine running the daemon). You probably don't want that. -In most cases, it's best to put each Dockerfile in an empty directory, and then add only -the files needed for building that Dockerfile to that directory. To further speed up the -build, you can exclude files and directories by adding a `.dockerignore` file to the same -directory. +In most cases, it's best to put each Dockerfile in an empty directory. Then, +only add the files needed for building the Dockerfile to the directory. To +increase the build's performance, you can exclude files and directories by +adding a `.dockerignore` file to the directory. For information about how to +[create a `.dockerignore` file](#the-dockerignore-file) on this page. You can specify a repository and tag at which to save the new image if the build succeeds: @@ -169,43 +170,67 @@ will result in `def` having a value of `hello`, not `bye`. However, `ghi` will have a value of `bye` because it is not part of the same command that set `abc` to `bye`. -## The `.dockerignore` file +### .dockerignore file -If a file named `.dockerignore` exists in the source repository, then it -is interpreted as a newline-separated list of exclusion patterns. -Exclusion patterns match files or directories relative to the source repository -that will be excluded from the context. Globbing is done using Go's +If a file named `.dockerignore` exists in the root of `PATH`, then Docker +interprets it as a newline-separated list of exclusion patterns. Docker excludes +files or directories relative to `PATH` that match these exclusion patterns. If +there are any `.dockerignore` files in `PATH` subdirectories, Docker treats +them as normal files. + +Filepaths in `.dockerignore` are absolute with the current directory as the +root. Wildcards are allowed but the search is not recursive. Globbing (file name +expansion) is done using Go's [filepath.Match](http://golang.org/pkg/path/filepath#Match) rules. -> **Note**: -> The `.dockerignore` file can even be used to ignore the `Dockerfile` and -> `.dockerignore` files. This might be useful if you are copying files from -> the root of the build context into your new container but do not want to -> include the `Dockerfile` or `.dockerignore` files (e.g. `ADD . /someDir/`). +You can specify exceptions to exclusion rules. To do this, simply prefix a +pattern with an `!` (exclamation mark) in the same way you would in a +`.gitignore` file. Currently there is no support for regular expressions. +Formats like `[^temp*]` are ignored. -The following example shows the use of the `.dockerignore` file to exclude the -`.git` directory from the context. Its effect can be seen in the changed size of -the uploaded context. +The following is an example `.dockerignore` file: + +``` + */temp* + */*/temp* + temp? + *.md + !LICENCSE.md +``` + +This file causes the following build behavior: + +| Rule | Behavior | +|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `*/temp*` | Exclude all files with names starting with`temp` in any subdirectory below the root directory. For example, a file named`/somedir/temporary.txt` is ignored. | +| `*/*/temp*` | Exclude files starting with name `temp` from any subdirectory that is two levels below the root directory. For example, the file `/somedir/subdir/temporary.txt` is ignored. | +| `temp?` | Exclude the files that match the pattern in the root directory. For example, the files `tempa`, `tempb` in the root directory are ignored. | +| `*.md ` | Exclude all markdown files. | +| `!LICENSE.md` | Exception to the exclude all Markdown files is this file, `LICENSE.md`, include this file in the build. | + +The placement of `!` exception rules influences the matching algorithm; the +last line of the `.dockerignore` that matches a particular file determines +whether it is included or excluded. In the above example, the `LICENSE.md` file +matches both the `*.md` and `!LICENSE.md` rule. If you reverse the lines in the +example: + +``` + */temp* + */*/temp* + temp? + !LICENCSE.md + *.md +``` + +The build would exclude `LICENSE.md` because the last `*.md` rule adds all +Markdown files back onto the ignore list. The `!LICENSE.md` rule has no effect +because the subsequent `*.md` rule overrides it. + +You can even use the `.dockerignore` file to ignore the `Dockerfile` and +`.dockerignore` files. This is useful if you are copying files from the root of +the build context into your new container but do not want to include the +`Dockerfile` or `.dockerignore` files (e.g. `ADD . /someDir/`). - $ docker build . - Uploading context 18.829 MB - Uploading context - Step 0 : FROM busybox - ---> 769b9341d937 - Step 1 : CMD echo Hello World - ---> Using cache - ---> 99cc1ad10469 - Successfully built 99cc1ad10469 - $ echo ".git" > .dockerignore - $ docker build . - Uploading context 6.76 MB - Uploading context - Step 0 : FROM busybox - ---> 769b9341d937 - Step 1 : CMD echo Hello World - ---> Using cache - ---> 99cc1ad10469 - Successfully built 99cc1ad10469 ## FROM diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 26659c8ffa..597324eae0 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -653,6 +653,26 @@ If you use STDIN or specify a `URL`, the system places the contents into a file called `Dockerfile`, and any `-f`, `--file` option is ignored. In this scenario, there is no context. +By default the `docker build` command will look for a `Dockerfile` at the +root of the build context. The `-f`, `--file`, option lets you specify +the path to an alternative file to use instead. This is useful +in cases where the same set of files are used for multiple builds. The path +must be to a file within the build context. If a relative path is specified +then it must to be relative to the current directory. + +In most cases, it's best to put each Dockerfile in an empty directory. Then, add +to that directory only the files needed for building the Dockerfile. To increase +the build's performance, you can exclude files and directories by adding a +`.dockerignore` file to that directory as well. For information on creating one, +see the [.dockerignore file](../../reference/builder/#dockerignore-file). + +If the Docker client loses connection to the daemon, the build is canceled. +This happens if you interrupt the Docker client with `ctrl-c` or if the Docker +client is killed for any reason. + +> **Note:** Currently only the "run" phase of the build can be canceled until +> pull cancelation is implemented). + ### Return code On a successful build, a return code of success `0` will be returned. @@ -673,55 +693,11 @@ INFO[0000] The command [/bin/sh -c exit 13] returned a non-zero code: 13 $ echo $? 1 ``` - -### .dockerignore file - -If a file named `.dockerignore` exists in the root of `PATH` then it -is interpreted as a newline-separated list of exclusion patterns. -Exclusion patterns match files or directories relative to `PATH` that -will be excluded from the context. Globbing is done using Go's -[filepath.Match](http://golang.org/pkg/path/filepath#Match) rules. - -Please note that `.dockerignore` files in other subdirectories are -considered as normal files. Filepaths in `.dockerignore` are absolute with -the current directory as the root. Wildcards are allowed but the search -is not recursive. - -#### Example .dockerignore file - */temp* - */*/temp* - temp? - -The first line above `*/temp*`, would ignore all files with names starting with -`temp` from any subdirectory below the root directory. For example, a file named -`/somedir/temporary.txt` would be ignored. The second line `*/*/temp*`, will -ignore files starting with name `temp` from any subdirectory that is two levels -below the root directory. For example, the file `/somedir/subdir/temporary.txt` -would get ignored in this case. The last line in the above example `temp?` -will ignore the files that match the pattern from the root directory. -For example, the files `tempa`, `tempb` are ignored from the root directory. -Currently there is no support for regular expressions. Formats -like `[^temp*]` are ignored. - -By default the `docker build` command will look for a `Dockerfile` at the -root of the build context. The `-f`, `--file`, option lets you specify -the path to an alternative file to use instead. This is useful -in cases where the same set of files are used for multiple builds. The path -must be to a file within the build context. If a relative path is specified -then it must to be relative to the current directory. - -If the Docker client loses connection to the daemon, the build is canceled. -This happens if you interrupt the Docker client with `ctrl-c` or if the Docker -client is killed for any reason. - -> **Note:** Currently only the "run" phase of the build can be canceled until -> pull cancelation is implemented). - See also: [*Dockerfile Reference*](/reference/builder). -#### Examples +### Examples $ docker build . Uploading context 10240 bytes @@ -790,7 +766,8 @@ affect the build cache. This example shows the use of the `.dockerignore` file to exclude the `.git` directory from the context. Its effect can be seen in the changed size of the -uploaded context. +uploaded context. The builder reference contains detailed information on +[creating a .dockerignore file](../../builder/#dockerignore-file) $ docker build -t vieux/apache:2.0 . diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 695e4cd6e6..3343427831 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -3427,20 +3427,29 @@ func (s *DockerSuite) TestBuildDockerignore(c *check.C) { RUN [[ ! -e /bla/src/_vendor ]] RUN [[ ! -e /bla/.gitignore ]] RUN [[ ! -e /bla/README.md ]] + RUN [[ ! -e /bla/dir/foo ]] + RUN [[ ! -e /bla/foo ]] RUN [[ ! -e /bla/.git ]]` ctx, err := fakeContext(dockerfile, map[string]string{ "Makefile": "all:", ".git/HEAD": "ref: foo", "src/x.go": "package main", "src/_vendor/v.go": "package main", + "dir/foo": "", ".gitignore": "", "README.md": "readme", - ".dockerignore": ".git\npkg\n.gitignore\nsrc/_vendor\n*.md", + ".dockerignore": ` +.git +pkg +.gitignore +src/_vendor +*.md +dir`, }) - defer ctx.Close() if err != nil { c.Fatal(err) } + defer ctx.Close() if _, err := buildImageFromContext(name, ctx, true); err != nil { c.Fatal(err) } @@ -3467,6 +3476,55 @@ func (s *DockerSuite) TestBuildDockerignoreCleanPaths(c *check.C) { } } +func (s *DockerSuite) TestBuildDockerignoreExceptions(c *check.C) { + name := "testbuilddockerignoreexceptions" + defer deleteImages(name) + dockerfile := ` + FROM busybox + ADD . /bla + RUN [[ -f /bla/src/x.go ]] + RUN [[ -f /bla/Makefile ]] + RUN [[ ! -e /bla/src/_vendor ]] + RUN [[ ! -e /bla/.gitignore ]] + RUN [[ ! -e /bla/README.md ]] + RUN [[ -e /bla/dir/dir/foo ]] + RUN [[ ! -e /bla/dir/foo1 ]] + RUN [[ -f /bla/dir/e ]] + RUN [[ -f /bla/dir/e-dir/foo ]] + RUN [[ ! -e /bla/foo ]] + RUN [[ ! -e /bla/.git ]]` + ctx, err := fakeContext(dockerfile, map[string]string{ + "Makefile": "all:", + ".git/HEAD": "ref: foo", + "src/x.go": "package main", + "src/_vendor/v.go": "package main", + "dir/foo": "", + "dir/foo1": "", + "dir/dir/f1": "", + "dir/dir/foo": "", + "dir/e": "", + "dir/e-dir/foo": "", + ".gitignore": "", + "README.md": "readme", + ".dockerignore": ` +.git +pkg +.gitignore +src/_vendor +*.md +dir +!dir/e* +!dir/dir/foo`, + }) + if err != nil { + c.Fatal(err) + } + defer ctx.Close() + if _, err := buildImageFromContext(name, ctx, true); err != nil { + c.Fatal(err) + } +} + func (s *DockerSuite) TestBuildDockerignoringDockerfile(c *check.C) { name := "testbuilddockerignoredockerfile" dockerfile := ` @@ -3607,6 +3665,7 @@ func (s *DockerSuite) TestBuildDockerignoringWholeDir(c *check.C) { ctx, err := fakeContext(dockerfile, map[string]string{ "Dockerfile": "FROM scratch", "Makefile": "all:", + ".gitignore": "", ".dockerignore": ".*\n", }) defer ctx.Close() diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 7f18897503..4d8d260087 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -391,6 +391,13 @@ func Tar(path string, compression Compression) (io.ReadCloser, error) { // TarWithOptions creates an archive from the directory at `path`, only including files whose relative // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { + + patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) + + if err != nil { + return nil, err + } + pipeReader, pipeWriter := io.Pipe() compressWriter, err := CompressStream(pipeWriter, options.Compression) @@ -441,7 +448,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) // is asking for that file no matter what - which is true // for some files, like .dockerignore and Dockerfile (sometimes) if include != relFilePath { - skip, err = fileutils.Matches(relFilePath, options.ExcludePatterns) + skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) if err != nil { logrus.Debugf("Error matching %s", relFilePath, err) return err @@ -449,7 +456,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } if skip { - if f.IsDir() { + if !exceptions && f.IsDir() { return filepath.SkipDir } return nil diff --git a/pkg/fileutils/fileutils.go b/pkg/fileutils/fileutils.go index ef2a6523dc..d1e130d0ad 100644 --- a/pkg/fileutils/fileutils.go +++ b/pkg/fileutils/fileutils.go @@ -1,33 +1,120 @@ package fileutils import ( + "errors" "fmt" "io" "io/ioutil" "os" "path/filepath" + "strings" "github.com/Sirupsen/logrus" ) -// Matches returns true if relFilePath matches any of the patterns -func Matches(relFilePath string, patterns []string) (bool, error) { - for _, exclude := range patterns { - matched, err := filepath.Match(exclude, relFilePath) +func Exclusion(pattern string) bool { + return pattern[0] == '!' +} + +func Empty(pattern string) bool { + return pattern == "" +} + +// Cleanpatterns takes a slice of patterns returns a new +// slice of patterns cleaned with filepath.Clean, stripped +// of any empty patterns and lets the caller know whether the +// slice contains any exception patterns (prefixed with !). +func CleanPatterns(patterns []string) ([]string, [][]string, bool, error) { + // Loop over exclusion patterns and: + // 1. Clean them up. + // 2. Indicate whether we are dealing with any exception rules. + // 3. Error if we see a single exclusion marker on it's own (!). + cleanedPatterns := []string{} + patternDirs := [][]string{} + exceptions := false + for _, pattern := range patterns { + // Eliminate leading and trailing whitespace. + pattern = strings.TrimSpace(pattern) + if Empty(pattern) { + continue + } + if Exclusion(pattern) { + if len(pattern) == 1 { + logrus.Errorf("Illegal exclusion pattern: %s", pattern) + return nil, nil, false, errors.New("Illegal exclusion pattern: !") + } + exceptions = true + } + pattern = filepath.Clean(pattern) + cleanedPatterns = append(cleanedPatterns, pattern) + if Exclusion(pattern) { + pattern = pattern[1:] + } + patternDirs = append(patternDirs, strings.Split(pattern, "/")) + } + + return cleanedPatterns, patternDirs, exceptions, nil +} + +// Matches returns true if file matches any of the patterns +// and isn't excluded by any of the subsequent patterns. +func Matches(file string, patterns []string) (bool, error) { + file = filepath.Clean(file) + + if file == "." { + // Don't let them exclude everything, kind of silly. + return false, nil + } + + patterns, patDirs, _, err := CleanPatterns(patterns) + if err != nil { + return false, err + } + + return OptimizedMatches(file, patterns, patDirs) +} + +// Matches is basically the same as fileutils.Matches() but optimized for archive.go. +// It will assume that the inputs have been preprocessed and therefore the function +// doen't need to do as much error checking and clean-up. This was done to avoid +// repeating these steps on each file being checked during the archive process. +// The more generic fileutils.Matches() can't make these assumptions. +func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool, error) { + matched := false + parentPath := filepath.Dir(file) + parentPathDirs := strings.Split(parentPath, "/") + + for i, pattern := range patterns { + negative := false + + if Exclusion(pattern) { + negative = true + pattern = pattern[1:] + } + + match, err := filepath.Match(pattern, file) if err != nil { - logrus.Errorf("Error matching: %s (pattern: %s)", relFilePath, exclude) + logrus.Errorf("Error matching: %s (pattern: %s)", file, pattern) return false, err } - if matched { - if filepath.Clean(relFilePath) == "." { - logrus.Errorf("Can't exclude whole path, excluding pattern: %s", exclude) - continue + + if !match && parentPath != "." { + // Check to see if the pattern matches one of our parent dirs. + if len(patDirs[i]) <= len(parentPathDirs) { + match, _ = filepath.Match(strings.Join(patDirs[i], "/"), + strings.Join(parentPathDirs[:len(patDirs[i])], "/")) } - logrus.Debugf("Skipping excluded path: %s", relFilePath) - return true, nil + } + + if match { + matched = !negative } } - return false, nil + + if matched { + logrus.Debugf("Skipping excluded path: %s", file) + } + return matched, nil } func CopyFile(src, dst string) (int64, error) { diff --git a/pkg/fileutils/fileutils_test.go b/pkg/fileutils/fileutils_test.go index 16d00d7b95..ce19ffce4a 100644 --- a/pkg/fileutils/fileutils_test.go +++ b/pkg/fileutils/fileutils_test.go @@ -79,3 +79,142 @@ func TestReadSymlinkedDirectoryToFile(t *testing.T) { t.Errorf("failed to remove symlink: %s", err) } } + +func TestWildcardMatches(t *testing.T) { + match, _ := Matches("fileutils.go", []string{"*"}) + if match != true { + t.Errorf("failed to get a wildcard match, got %v", match) + } +} + +// A simple pattern match should return true. +func TestPatternMatches(t *testing.T) { + match, _ := Matches("fileutils.go", []string{"*.go"}) + if match != true { + t.Errorf("failed to get a match, got %v", match) + } +} + +// An exclusion followed by an inclusion should return true. +func TestExclusionPatternMatchesPatternBefore(t *testing.T) { + match, _ := Matches("fileutils.go", []string{"!fileutils.go", "*.go"}) + if match != true { + t.Errorf("failed to get true match on exclusion pattern, got %v", match) + } +} + +// A folder pattern followed by an exception should return false. +func TestPatternMatchesFolderExclusions(t *testing.T) { + match, _ := Matches("docs/README.md", []string{"docs", "!docs/README.md"}) + if match != false { + t.Errorf("failed to get a false match on exclusion pattern, got %v", match) + } +} + +// A folder pattern followed by an exception should return false. +func TestPatternMatchesFolderWithSlashExclusions(t *testing.T) { + match, _ := Matches("docs/README.md", []string{"docs/", "!docs/README.md"}) + if match != false { + t.Errorf("failed to get a false match on exclusion pattern, got %v", match) + } +} + +// A folder pattern followed by an exception should return false. +func TestPatternMatchesFolderWildcardExclusions(t *testing.T) { + match, _ := Matches("docs/README.md", []string{"docs/*", "!docs/README.md"}) + if match != false { + t.Errorf("failed to get a false match on exclusion pattern, got %v", match) + } +} + +// A pattern followed by an exclusion should return false. +func TestExclusionPatternMatchesPatternAfter(t *testing.T) { + match, _ := Matches("fileutils.go", []string{"*.go", "!fileutils.go"}) + if match != false { + t.Errorf("failed to get false match on exclusion pattern, got %v", match) + } +} + +// A filename evaluating to . should return false. +func TestExclusionPatternMatchesWholeDirectory(t *testing.T) { + match, _ := Matches(".", []string{"*.go"}) + if match != false { + t.Errorf("failed to get false match on ., got %v", match) + } +} + +// A single ! pattern should return an error. +func TestSingleExclamationError(t *testing.T) { + _, err := Matches("fileutils.go", []string{"!"}) + if err == nil { + t.Errorf("failed to get an error for a single exclamation point, got %v", err) + } +} + +// A string preceded with a ! should return true from Exclusion. +func TestExclusion(t *testing.T) { + exclusion := Exclusion("!") + if !exclusion { + t.Errorf("failed to get true for a single !, got %v", exclusion) + } +} + +// An empty string should return true from Empty. +func TestEmpty(t *testing.T) { + empty := Empty("") + if !empty { + t.Errorf("failed to get true for an empty string, got %v", empty) + } +} + +func TestCleanPatterns(t *testing.T) { + cleaned, _, _, _ := CleanPatterns([]string{"docs", "config"}) + if len(cleaned) != 2 { + t.Errorf("expected 2 element slice, got %v", len(cleaned)) + } +} + +func TestCleanPatternsStripEmptyPatterns(t *testing.T) { + cleaned, _, _, _ := CleanPatterns([]string{"docs", "config", ""}) + if len(cleaned) != 2 { + t.Errorf("expected 2 element slice, got %v", len(cleaned)) + } +} + +func TestCleanPatternsExceptionFlag(t *testing.T) { + _, _, exceptions, _ := CleanPatterns([]string{"docs", "!docs/README.md"}) + if !exceptions { + t.Errorf("expected exceptions to be true, got %v", exceptions) + } +} + +func TestCleanPatternsLeadingSpaceTrimmed(t *testing.T) { + _, _, exceptions, _ := CleanPatterns([]string{"docs", " !docs/README.md"}) + if !exceptions { + t.Errorf("expected exceptions to be true, got %v", exceptions) + } +} + +func TestCleanPatternsTrailingSpaceTrimmed(t *testing.T) { + _, _, exceptions, _ := CleanPatterns([]string{"docs", "!docs/README.md "}) + if !exceptions { + t.Errorf("expected exceptions to be true, got %v", exceptions) + } +} + +func TestCleanPatternsErrorSingleException(t *testing.T) { + _, _, _, err := CleanPatterns([]string{"!"}) + if err == nil { + t.Errorf("expected error on single exclamation point, got %v", err) + } +} + +func TestCleanPatternsFolderSplit(t *testing.T) { + _, dirs, _, _ := CleanPatterns([]string{"docs/config/CONFIG.md"}) + if dirs[0][0] != "docs" { + t.Errorf("expected first element in dirs slice to be docs, got %v", dirs[0][1]) + } + if dirs[0][1] != "config" { + t.Errorf("expected first element in dirs slice to be config, got %v", dirs[0][1]) + } +}