Merge pull request #3295 from alexlarsson/implement-untar

Implement UnTar via archive/tar

archive/archive.go

@@ -13,6 +13,8 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
+	"syscall"
 )
 
 type Archive io.Reader
@@ -124,6 +126,84 @@ func (compression *Compression) Extension() string {
 	return ""
 }
 
+func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) error {
+	switch hdr.Typeflag {
+	case tar.TypeDir:
+		// Create directory unless it exists as a directory already.
+		// In that case we just want to merge the two
+		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
+			if err := os.Mkdir(path, os.FileMode(hdr.Mode)); err != nil {
+				return err
+			}
+		}
+
+	case tar.TypeReg, tar.TypeRegA:
+		// Source is regular file
+		file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(file, reader); err != nil {
+			file.Close()
+			return err
+		}
+		file.Close()
+
+	case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
+		mode := uint32(hdr.Mode & 07777)
+		switch hdr.Typeflag {
+		case tar.TypeBlock:
+			mode |= syscall.S_IFBLK
+		case tar.TypeChar:
+			mode |= syscall.S_IFCHR
+		case tar.TypeFifo:
+			mode |= syscall.S_IFIFO
+		}
+
+		if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
+			return err
+		}
+
+	case tar.TypeLink:
+		if err := os.Link(filepath.Join(extractDir, hdr.Linkname), path); err != nil {
+			return err
+		}
+
+	case tar.TypeSymlink:
+		if err := os.Symlink(hdr.Linkname, path); err != nil {
+			return err
+		}
+
+	default:
+		return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag)
+	}
+
+	if err := syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
+		return err
+	}
+
+	// There is no LChmod, so ignore mode for symlink. Also, this
+	// must happen after chown, as that can modify the file mode
+	if hdr.Typeflag != tar.TypeSymlink {
+		if err := syscall.Chmod(path, uint32(hdr.Mode&07777)); err != nil {
+			return err
+		}
+	}
+
+	ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+	// syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and
+	if hdr.Typeflag != tar.TypeSymlink {
+		if err := syscall.UtimesNano(path, ts); err != nil {
+			return err
+		}
+	} else {
+		if err := LUtimesNano(path, ts); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
 func Tar(path string, compression Compression) (io.Reader, error) {
@@ -206,45 +286,92 @@ func TarFilter(path string, options *TarOptions) (io.Reader, error) {
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, path string, options *TarOptions) error {
+func Untar(archive io.Reader, dest string, options *TarOptions) error {
 	if archive == nil {
 		return fmt.Errorf("Empty archive")
 	}
 
-	buf := make([]byte, 10)
+	archive, err := DecompressStream(archive)
-	totalN := 0
-	for totalN < 10 {
-		n, err := archive.Read(buf[totalN:])
 	if err != nil {
-			if err == io.EOF {
-				return fmt.Errorf("Tarball too short")
-			}
 		return err
 	}
-		totalN += n
+
-		utils.Debugf("[tar autodetect] n: %d", n)
+	tr := tar.NewReader(archive)
+
+	var dirs []*tar.Header
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return err
 		}
-
-	compression := DetectCompression(buf)
-
-	utils.Debugf("Archive compression detected: %s", compression.Extension())
-	args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()}
 
 		if options != nil {
+			excludeFile := false
 			for _, exclude := range options.Excludes {
-			args = append(args, fmt.Sprintf("--exclude=%s", exclude))
+				if strings.HasPrefix(hdr.Name, exclude) {
+					excludeFile = true
+					break
+				}
+			}
+			if excludeFile {
+				continue
 			}
 		}
 
-	cmd := exec.Command("tar", args...)
+		// Normalize name, for safety and for a simple is-root check
-	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
+		hdr.Name = filepath.Clean(hdr.Name)
-	// Hardcode locale environment for predictable outcome regardless of host configuration.
+
-	//   (see https://github.com/dotcloud/docker/issues/355)
+		if !strings.HasSuffix(hdr.Name, "/") {
-	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
+			// Not the root directory, ensure that the parent directory exists
-	output, err := cmd.CombinedOutput()
+			parent := filepath.Dir(hdr.Name)
+			parentPath := filepath.Join(dest, parent)
+			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
+				err = os.MkdirAll(parentPath, 600)
 				if err != nil {
-		return fmt.Errorf("%s: %s", err, output)
+					return err
 				}
+			}
+		}
+
+		path := filepath.Join(dest, hdr.Name)
+
+		// If path exits we almost always just want to remove and replace it
+		// The only exception is when it is a directory *and* the file from
+		// the layer is also a directory. Then we want to merge them (i.e.
+		// just apply the metadata from the layer).
+		if fi, err := os.Lstat(path); err == nil {
+			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
+				if err := os.RemoveAll(path); err != nil {
+					return err
+				}
+			}
+		}
+
+		if err := createTarFile(path, dest, hdr, tr); err != nil {
+			return err
+		}
+
+		// Directory mtimes must be handled at the end to avoid further
+		// file creation in them to modify the directory mtime
+		if hdr.Typeflag == tar.TypeDir {
+			dirs = append(dirs, hdr)
+		}
+	}
+
+	for _, hdr := range dirs {
+		path := filepath.Join(dest, hdr.Name)
+		ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+		if err := syscall.UtimesNano(path, ts); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

archive/diff.go

@@ -2,7 +2,6 @@ package archive
 
 import (
 	"archive/tar"
-	"github.com/dotcloud/docker/utils"
 	"io"
 	"os"
 	"path/filepath"
@@ -89,95 +88,22 @@ func ApplyLayer(dest string, layer Archive) error {
 			// The only exception is when it is a directory *and* the file from
 			// the layer is also a directory. Then we want to merge them (i.e.
 			// just apply the metadata from the layer).
-			hasDir := false
 			if fi, err := os.Lstat(path); err == nil {
-				if fi.IsDir() && hdr.Typeflag == tar.TypeDir {
+				if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
-					hasDir = true
-				} else {
 					if err := os.RemoveAll(path); err != nil {
 						return err
 					}
 				}
 			}
 
-			switch hdr.Typeflag {
+			if err := createTarFile(path, dest, hdr, tr); err != nil {
-			case tar.TypeDir:
-				if !hasDir {
-					err = os.Mkdir(path, os.FileMode(hdr.Mode))
-					if err != nil {
 				return err
 			}
-				}
+
+			// Directory mtimes must be handled at the end to avoid further
+			// file creation in them to modify the directory mtime
+			if hdr.Typeflag == tar.TypeDir {
 				dirs = append(dirs, hdr)
-
-			case tar.TypeReg, tar.TypeRegA:
-				// Source is regular file
-				file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
-				if err != nil {
-					return err
-				}
-				if _, err := io.Copy(file, tr); err != nil {
-					file.Close()
-					return err
-				}
-				file.Close()
-
-			case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
-				mode := uint32(hdr.Mode & 07777)
-				switch hdr.Typeflag {
-				case tar.TypeBlock:
-					mode |= syscall.S_IFBLK
-				case tar.TypeChar:
-					mode |= syscall.S_IFCHR
-				case tar.TypeFifo:
-					mode |= syscall.S_IFIFO
-				}
-
-				if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
-					return err
-				}
-
-			case tar.TypeLink:
-				if err := os.Link(filepath.Join(dest, hdr.Linkname), path); err != nil {
-					return err
-				}
-
-			case tar.TypeSymlink:
-				if err := os.Symlink(hdr.Linkname, path); err != nil {
-					return err
-				}
-
-			default:
-				utils.Debugf("unhandled type %d\n", hdr.Typeflag)
-			}
-
-			if err = syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
-				return err
-			}
-
-			// There is no LChmod, so ignore mode for symlink. Also, this
-			// must happen after chown, as that can modify the file mode
-			if hdr.Typeflag != tar.TypeSymlink {
-				err = syscall.Chmod(path, uint32(hdr.Mode&07777))
-				if err != nil {
-					return err
-				}
-			}
-
-			// Directories must be handled at the end to avoid further
-			// file creation in them to modify the mtime
-			if hdr.Typeflag != tar.TypeDir {
-				ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
-				// syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and
-				if hdr.Typeflag != tar.TypeSymlink {
-					if err := syscall.UtimesNano(path, ts); err != nil {
-						return err
-					}
-				} else {
-					if err := LUtimesNano(path, ts); err != nil {
-						return err
-					}
-				}
 			}
 		}
 	}