Implement TarFilter in go, rather than calling out to tar

This uses a plain filepath.Walk + addTarFile to create a tar file,
optionially compressing it with gzip.

Unfortunately go only has gzip compression support, not bzip2 or xz.
However, this is not a regression, as docker currently uses *no*
compression for TarFilter(). The only compression of tarfiles
currently happens in utils/tarsum.go, and that manually does gzip
compression.

Docker-DCO-1.1-Signed-off-by: Alexander Larsson <alexl@redhat.com> (github: alexlarsson)

archive/archive.go

@@ -97,16 +97,20 @@ func DecompressStream(archive io.Reader) (io.Reader, error) {
 	}
 }
 
-func (compression *Compression) Flag() string {
+func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {
-	switch *compression {
+
-	case Bzip2:
+	switch compression {
-		return "j"
+	case Uncompressed:
+		return dest, nil
 	case Gzip:
-		return "z"
+		return gzip.NewWriter(dest), nil
-	case Xz:
+	case Bzip2, Xz:
-		return "J"
+		// archive/bzip2 does not support writing, and there is no xz support at all
+		// However, this is not a problem as docker only currently generates gzipped tars
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
+	default:
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
 	}
-	return ""
 }
 
 func (compression *Compression) Extension() string {
@@ -130,7 +134,7 @@ func addTarFile(path, name string, tw *tar.Writer) error {
 	}
 
 	link := ""
-	if fi.Mode() & os.ModeSymlink != 0 {
+	if fi.Mode()&os.ModeSymlink != 0 {
 		if link, err = os.Readlink(path); err != nil {
 			return err
 		}
@@ -274,19 +278,55 @@ func escapeName(name string) string {
 
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
-func TarFilter(path string, options *TarOptions) (io.Reader, error) {
+func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) {
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"}
+	pipeReader, pipeWriter := io.Pipe()
+
+	compressWriter, err := CompressStream(pipeWriter, options.Compression)
+	if err != nil {
+		return nil, err
+	}
+
+	tw := tar.NewWriter(compressWriter)
+
+	go func() {
+		// In general we log errors here but ignore them because
+		// during e.g. a diff operation the container can continue
+		// mutating the filesystem and we can see transient errors
+		// from this
+
 		if options.Includes == nil {
 			options.Includes = []string{"."}
 		}
-	args = append(args, "-c"+options.Compression.Flag())
 
-	files := ""
+		for _, include := range options.Includes {
-	for _, f := range options.Includes {
+			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
-		files = files + escapeName(f) + "\n"
+				if err != nil {
+					utils.Debugf("Tar: Can't stat file %s to tar: %s\n", srcPath, err)
+					return nil
 				}
 
-	return CmdStream(exec.Command(args[0], args[1:]...), bytes.NewBufferString(files))
+				relFilePath, err := filepath.Rel(srcPath, filePath)
+				if err != nil {
+					return nil
+				}
+
+				if err := addTarFile(filePath, relFilePath, tw); err != nil {
+					utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err)
+				}
+				return nil
+			})
+		}
+
+		// Make sure to check the error on Close.
+		if err := tw.Close(); err != nil {
+			utils.Debugf("Can't close tar writer: %s\n", err)
+		}
+		if err := compressWriter.Close(); err != nil {
+			utils.Debugf("Can't close compress writer: %s\n", err)
+		}
+	}()
+
+	return pipeReader, nil
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,

archive/archive_test.go

@@ -89,6 +89,16 @@ func tarUntar(t *testing.T, origin string, compression Compression) error {
 	if _, err := os.Stat(tmp); err != nil {
 		return err
 	}
+
+	changes, err := ChangesDirs(origin, tmp)
+	if err != nil {
+		return err
+	}
+
+	if len(changes) != 0 {
+		t.Fatalf("Unexpected differences after tarUntar: %v", changes)
+	}
+
 	return nil
 }
 
@@ -108,8 +118,6 @@ func TestTarUntar(t *testing.T) {
 	for _, c := range []Compression{
 		Uncompressed,
 		Gzip,
-		Bzip2,
-		Xz,
 	} {
 		if err := tarUntar(t, origin, c); err != nil {
 			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)