diff --git a/archive/archive.go b/archive/archive.go index 89a360c906..727e9289fa 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -23,10 +23,7 @@ type Compression int type TarOptions struct { Includes []string - Excludes []string - Recursive bool Compression Compression - CreateFiles []string } const ( @@ -66,7 +63,7 @@ func DetectCompression(source []byte) Compression { func xzDecompress(archive io.Reader) (io.Reader, error) { args := []string{"xz", "-d", "-c", "-q"} - return CmdStream(exec.Command(args[0], args[1:]...), archive, nil) + return CmdStream(exec.Command(args[0], args[1:]...), archive) } func DecompressStream(archive io.Reader) (io.Reader, error) { @@ -100,16 +97,20 @@ func DecompressStream(archive io.Reader) (io.Reader, error) { } } -func (compression *Compression) Flag() string { - switch *compression { - case Bzip2: - return "j" +func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) { + + switch compression { + case Uncompressed: + return dest, nil case Gzip: - return "z" - case Xz: - return "J" + return gzip.NewWriter(dest), nil + case Bzip2, Xz: + // archive/bzip2 does not support writing, and there is no xz support at all + // However, this is not a problem as docker only currently generates gzipped tars + return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) + default: + return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) } - return "" } func (compression *Compression) Extension() string { @@ -126,6 +127,59 @@ func (compression *Compression) Extension() string { return "" } +func addTarFile(path, name string, tw *tar.Writer) error { + fi, err := os.Lstat(path) + if err != nil { + return err + } + + link := "" + if fi.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(fi, link) + if err != nil { + return err + } + + if fi.IsDir() && !strings.HasSuffix(name, "/") { + name = name + "/" + } + + hdr.Name = name + + stat, ok := fi.Sys().(*syscall.Stat_t) + if ok { + // Currently go does not fill in the major/minors + if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK || + stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR { + hdr.Devmajor = int64(major(uint64(stat.Rdev))) + hdr.Devminor = int64(minor(uint64(stat.Rdev))) + } + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + if hdr.Typeflag == tar.TypeReg { + if file, err := os.Open(path); err != nil { + return err + } else { + _, err := io.Copy(tw, file) + if err != nil { + return err + } + file.Close() + } + } + + return nil +} + func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) error { switch hdr.Typeflag { case tar.TypeDir: @@ -207,7 +261,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) // Tar creates an archive from the directory at `path`, and returns it as a // stream of bytes. func Tar(path string, compression Compression) (io.Reader, error) { - return TarFilter(path, &TarOptions{Recursive: true, Compression: compression}) + return TarFilter(path, &TarOptions{Compression: compression}) } func escapeName(name string) string { @@ -228,57 +282,55 @@ func escapeName(name string) string { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. -func TarFilter(path string, options *TarOptions) (io.Reader, error) { - args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"} - if options.Includes == nil { - options.Includes = []string{"."} - } - args = append(args, "-c"+options.Compression.Flag()) +func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) { + pipeReader, pipeWriter := io.Pipe() - for _, exclude := range options.Excludes { - args = append(args, fmt.Sprintf("--exclude=%s", exclude)) + compressWriter, err := CompressStream(pipeWriter, options.Compression) + if err != nil { + return nil, err } - if !options.Recursive { - args = append(args, "--no-recursion") - } + tw := tar.NewWriter(compressWriter) - files := "" - for _, f := range options.Includes { - files = files + escapeName(f) + "\n" - } + go func() { + // In general we log errors here but ignore them because + // during e.g. a diff operation the container can continue + // mutating the filesystem and we can see transient errors + // from this - tmpDir := "" - - if options.CreateFiles != nil { - var err error // Can't use := here or we override the outer tmpDir - tmpDir, err = ioutil.TempDir("", "docker-tar") - if err != nil { - return nil, err + if options.Includes == nil { + options.Includes = []string{"."} } - files = files + "-C" + tmpDir + "\n" - for _, f := range options.CreateFiles { - path := filepath.Join(tmpDir, f) - err := os.MkdirAll(filepath.Dir(path), 0600) - if err != nil { - return nil, err - } + for _, include := range options.Includes { + filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { + if err != nil { + utils.Debugf("Tar: Can't stat file %s to tar: %s\n", srcPath, err) + return nil + } - if file, err := os.OpenFile(path, os.O_CREATE, 0600); err != nil { - return nil, err - } else { - file.Close() - } - files = files + escapeName(f) + "\n" - } - } + relFilePath, err := filepath.Rel(srcPath, filePath) + if err != nil { + return nil + } - return CmdStream(exec.Command(args[0], args[1:]...), bytes.NewBufferString(files), func() { - if tmpDir != "" { - _ = os.RemoveAll(tmpDir) + if err := addTarFile(filePath, relFilePath, tw); err != nil { + utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err) + } + return nil + }) } - }) + + // Make sure to check the error on Close. + if err := tw.Close(); err != nil { + utils.Debugf("Can't close tar writer: %s\n", err) + } + if err := compressWriter.Close(); err != nil { + utils.Debugf("Can't close compress writer: %s\n", err) + } + }() + + return pipeReader, nil } // Untar reads a stream of bytes from `archive`, parses it as a tar archive, @@ -311,19 +363,6 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { return err } - if options != nil { - excludeFile := false - for _, exclude := range options.Excludes { - if strings.HasPrefix(hdr.Name, exclude) { - excludeFile = true - break - } - } - if excludeFile { - continue - } - } - // Normalize name, for safety and for a simple is-root check hdr.Name = filepath.Clean(hdr.Name) @@ -378,9 +417,9 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { // TarUntar is a convenience function which calls Tar and Untar, with // the output of one piped into the other. If either Tar or Untar fails, // TarUntar aborts and returns the error. -func TarUntar(src string, filter []string, dst string) error { - utils.Debugf("TarUntar(%s %s %s)", src, filter, dst) - archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed, Includes: filter, Recursive: true}) +func TarUntar(src string, dst string) error { + utils.Debugf("TarUntar(%s %s)", src, dst) + archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed}) if err != nil { return err } @@ -417,7 +456,7 @@ func CopyWithTar(src, dst string) error { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) - return TarUntar(src, nil, dst) + return TarUntar(src, dst) } // CopyFileWithTar emulates the behavior of the 'cp' command-line @@ -480,13 +519,10 @@ func CopyFileWithTar(src, dst string) (err error) { // CmdStream executes a command, and returns its stdout as a stream. // If the command fails to run or doesn't complete successfully, an error // will be returned, including anything written on stderr. -func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) { +func CmdStream(cmd *exec.Cmd, input io.Reader) (io.Reader, error) { if input != nil { stdin, err := cmd.StdinPipe() if err != nil { - if atEnd != nil { - atEnd() - } return nil, err } // Write stdin if any @@ -497,16 +533,10 @@ func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) } stdout, err := cmd.StdoutPipe() if err != nil { - if atEnd != nil { - atEnd() - } return nil, err } stderr, err := cmd.StderrPipe() if err != nil { - if atEnd != nil { - atEnd() - } return nil, err } pipeR, pipeW := io.Pipe() @@ -531,9 +561,6 @@ func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) } else { pipeW.Close() } - if atEnd != nil { - atEnd() - } }() // Run the command and return the pipe if err := cmd.Start(); err != nil { diff --git a/archive/archive_test.go b/archive/archive_test.go index 684d99dc14..a5629deff1 100644 --- a/archive/archive_test.go +++ b/archive/archive_test.go @@ -14,7 +14,7 @@ import ( func TestCmdStreamLargeStderr(t *testing.T) { cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello") - out, err := CmdStream(cmd, nil, nil) + out, err := CmdStream(cmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) } @@ -35,7 +35,7 @@ func TestCmdStreamLargeStderr(t *testing.T) { func TestCmdStreamBad(t *testing.T) { badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") - out, err := CmdStream(badCmd, nil, nil) + out, err := CmdStream(badCmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) } @@ -50,7 +50,7 @@ func TestCmdStreamBad(t *testing.T) { func TestCmdStreamGood(t *testing.T) { cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0") - out, err := CmdStream(cmd, nil, nil) + out, err := CmdStream(cmd, nil) if err != nil { t.Fatal(err) } @@ -89,6 +89,16 @@ func tarUntar(t *testing.T, origin string, compression Compression) error { if _, err := os.Stat(tmp); err != nil { return err } + + changes, err := ChangesDirs(origin, tmp) + if err != nil { + return err + } + + if len(changes) != 0 { + t.Fatalf("Unexpected differences after tarUntar: %v", changes) + } + return nil } @@ -108,8 +118,6 @@ func TestTarUntar(t *testing.T) { for _, c := range []Compression{ Uncompressed, Gzip, - Bzip2, - Xz, } { if err := tarUntar(t, origin, c); err != nil { t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) diff --git a/archive/changes.go b/archive/changes.go index c67bec8ce2..25406f5cec 100644 --- a/archive/changes.go +++ b/archive/changes.go @@ -347,70 +347,12 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { } } else { path := filepath.Join(dir, change.Path) - - var stat syscall.Stat_t - if err := syscall.Lstat(path, &stat); err != nil { - utils.Debugf("Can't stat source file: %s\n", err) - continue - } - - mtim := getLastModification(&stat) - atim := getLastAccess(&stat) - hdr := &tar.Header{ - Name: change.Path[1:], - Mode: int64(stat.Mode & 07777), - Uid: int(stat.Uid), - Gid: int(stat.Gid), - ModTime: time.Unix(int64(mtim.Sec), int64(mtim.Nsec)), - AccessTime: time.Unix(int64(atim.Sec), int64(atim.Nsec)), - } - - if stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR { - hdr.Typeflag = tar.TypeDir - } else if stat.Mode&syscall.S_IFLNK == syscall.S_IFLNK { - hdr.Typeflag = tar.TypeSymlink - if link, err := os.Readlink(path); err != nil { - utils.Debugf("Can't readlink source file: %s\n", err) - continue - } else { - hdr.Linkname = link - } - } else if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK || - stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR { - if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK { - hdr.Typeflag = tar.TypeBlock - } else { - hdr.Typeflag = tar.TypeChar - } - hdr.Devmajor = int64(major(uint64(stat.Rdev))) - hdr.Devminor = int64(minor(uint64(stat.Rdev))) - } else if stat.Mode&syscall.S_IFIFO == syscall.S_IFIFO || - stat.Mode&syscall.S_IFSOCK == syscall.S_IFSOCK { - hdr.Typeflag = tar.TypeFifo - } else if stat.Mode&syscall.S_IFREG == syscall.S_IFREG { - hdr.Typeflag = tar.TypeReg - hdr.Size = stat.Size - } else { - utils.Debugf("Unknown file type: %s\n", path) - continue - } - - if err := tw.WriteHeader(hdr); err != nil { - utils.Debugf("Can't write tar header: %s\n", err) - } - if hdr.Typeflag == tar.TypeReg { - if file, err := os.Open(path); err != nil { - utils.Debugf("Can't open file: %s\n", err) - } else { - _, err := io.Copy(tw, file) - if err != nil { - utils.Debugf("Can't copy file: %s\n", err) - } - file.Close() - } + if err := addTarFile(path, change.Path[1:], tw); err != nil { + utils.Debugf("Can't add file %s to tar: %s\n", path, err) } } } + // Make sure to check the error on Close. if err := tw.Close(); err != nil { utils.Debugf("Can't close layer: %s\n", err) diff --git a/container.go b/container.go index a3fb5d6b39..5b068aa987 100644 --- a/container.go +++ b/container.go @@ -1473,7 +1473,6 @@ func (container *Container) Copy(resource string) (archive.Archive, error) { return archive.TarFilter(basePath, &archive.TarOptions{ Compression: archive.Uncompressed, Includes: filter, - Recursive: true, }) } diff --git a/graphdriver/aufs/aufs.go b/graphdriver/aufs/aufs.go index 2e9fa8298a..81cdd88134 100644 --- a/graphdriver/aufs/aufs.go +++ b/graphdriver/aufs/aufs.go @@ -225,7 +225,6 @@ func (a *Driver) Get(id string) (string, error) { // Returns an archive of the contents for the id func (a *Driver) Diff(id string) (archive.Archive, error) { return archive.TarFilter(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ - Recursive: true, Compression: archive.Uncompressed, }) } diff --git a/hack/PACKAGERS.md b/hack/PACKAGERS.md index 1dd039c3e3..2d869b6eda 100644 --- a/hack/PACKAGERS.md +++ b/hack/PACKAGERS.md @@ -120,7 +120,6 @@ The test suite will also download a small test container, so you will need inter To run properly, docker needs the following software to be installed at runtime: -* GNU Tar version 1.26 or later * iproute2 version 3.5 or later (build after 2012-05-21), and specifically the "ip" utility * iptables version 1.4 or later * The LXC utility scripts (http://lxc.sourceforge.net) version 0.8 or later diff --git a/integration/api_test.go b/integration/api_test.go index 8a9d64684f..6fac656c99 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -374,7 +374,7 @@ func TestGetContainersExport(t *testing.T) { } t.Fatal(err) } - if h.Name == "./test" { + if h.Name == "test" { found = true break }