Merge pull request #885 from dotcloud/remove_bsdtar

* Runtime: Remove bsdtar dependency

README.md

@@ -262,7 +262,7 @@ Setting up a dev environment
 Instructions that have been verified to work on Ubuntu 12.10,
 
 ```bash
-sudo apt-get -y install lxc wget bsdtar curl golang git
+sudo apt-get -y install lxc curl xz-utils golang git
 
 export GOPATH=~/go/
 export PATH=$GOPATH/bin:$PATH

archive.go

@@ -1,8 +1,10 @@
 package docker
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"os"
@@ -20,6 +22,37 @@ const (
 	Xz
 )
 
+func DetectCompression(source []byte) Compression {
+	for _, c := range source[:10] {
+		utils.Debugf("%x", c)
+	}
+
+	sourceLen := len(source)
+	for compression, m := range map[Compression][]byte{
+		Bzip2: {0x42, 0x5A, 0x68},
+		Gzip:  {0x1F, 0x8B, 0x08},
+		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
+	} {
+		fail := false
+		if len(m) > sourceLen {
+			utils.Debugf("Len too short")
+			continue
+		}
+		i := 0
+		for _, b := range m {
+			if b != source[i] {
+				fail = true
+				break
+			}
+			i++
+		}
+		if !fail {
+			return compression
+		}
+	}
+	return Uncompressed
+}
+
 func (compression *Compression) Flag() string {
 	switch *compression {
 	case Bzip2:
@@ -47,14 +80,23 @@ func (compression *Compression) Extension() string {
 }
 
 func Tar(path string, compression Compression) (io.Reader, error) {
-	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".")
+	return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), "."))
-	return CmdStream(cmd)
 }
 
 // FIXME: specify behavior when target path exists vs. doesn't exist.
 func Untar(archive io.Reader, path string) error {
-	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x")
+
-	cmd.Stdin = archive
+	bufferedArchive := bufio.NewReaderSize(archive, 10)
+	buf, err := bufferedArchive.Peek(10)
+	if err != nil {
+		return err
+	}
+	compression := DetectCompression(buf)
+
+	utils.Debugf("Archive compression detected: %s", compression.Extension())
+
+	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
+	cmd.Stdin = bufferedArchive
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
 	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}

archive_test.go

@@ -1,10 +1,13 @@
 package docker
 
 import (
+	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"testing"
 	"time"
 )
@@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) {
 	}
 }
 
-func TestTarUntar(t *testing.T) {
+func tarUntar(t *testing.T, origin string, compression Compression) error {
-	archive, err := Tar(".", Uncompressed)
+	archive, err := Tar(origin, compression)
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	buf := make([]byte, 10)
+	if _, err := archive.Read(buf); err != nil {
+		return err
+	}
+	archive = io.MultiReader(bytes.NewReader(buf), archive)
+
+	detectedCompression := DetectCompression(buf)
+	if detectedCompression.Extension() != compression.Extension() {
+		return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
+	}
+
 	tmp, err := ioutil.TempDir("", "docker-test-untar")
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 	defer os.RemoveAll(tmp)
 	if err := Untar(archive, tmp); err != nil {
-		t.Fatal(err)
+		return err
 	}
 	if _, err := os.Stat(tmp); err != nil {
-		t.Fatalf("Error stating %s: %s", tmp, err.Error())
+		return err
+	}
+	return nil
+}
+
+func TestTarUntar(t *testing.T) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(origin)
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range []Compression{
+		Uncompressed,
+		Gzip,
+		Bzip2,
+		Xz,
+	} {
+		if err := tarUntar(t, origin, c); err != nil {
+			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
+		}
 	}
 }

contrib/install.sh

@@ -8,7 +8,7 @@
 
 echo "Ensuring basic dependencies are installed..."
 apt-get -qq update
-apt-get -qq install lxc wget bsdtar
+apt-get -qq install lxc wget
 
 echo "Looking in /proc/filesystems to see if we have AUFS support..."
 if grep -q aufs /proc/filesystems

docs/sources/contributing/devenvironment.rst

@@ -33,7 +33,7 @@ Installation
     sudo apt-get install python-software-properties
     sudo add-apt-repository ppa:gophers/go
     sudo apt-get update
-    sudo apt-get -y install lxc wget bsdtar curl golang-stable git aufs-tools
+    sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools
 
     export GOPATH=~/go/
     export PATH=$GOPATH/bin:$PATH

docs/sources/installation/binaries.rst

@@ -30,8 +30,7 @@ Dependencies:
 * 3.8 Kernel (read more about :ref:`kernel`)
 * AUFS filesystem support
 * lxc
-* bsdtar
+* xz-utils
-
 
 Get the docker binary:
 ----------------------

graph_test.go

@@ -192,11 +192,19 @@ func TestDelete(t *testing.T) {
 	}
 	assertNImages(graph, t, 0)
 
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Test 2 create (same name) / 1 delete
 	img1, err := graph.Create(archive, nil, "Testing", "", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil {
 		t.Fatal(err)
 	}
@@ -212,6 +220,10 @@ func TestDelete(t *testing.T) {
 	}
 	assertNImages(graph, t, 1)
 
+	archive, err = fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Test delete twice (pull -> rm -> pull -> rm)
 	if err := graph.Register(archive, false, img1); err != nil {
 		t.Fatal(err)

hack/Vagrantfile

@@ -22,7 +22,7 @@ Vagrant::Config.run do |config|
   pkg_cmd = "touch #{DOCKER_PATH}; "
   # Install docker dependencies
   pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \
-    "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
+    "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
     "chown -R #{USER}.#{USER} #{GOPATH}; " \
     "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile"
   config.vm.provision :shell, :inline => pkg_cmd

testing/Vagrantfile

@@ -30,7 +30,7 @@ Vagrant::Config.run do |config|
     # Install docker dependencies
     pkg_cmd << "apt-get install -q -y python-software-properties; " \
       "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \
-      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; "
+      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; "
     # Activate new kernel
     pkg_cmd << "shutdown -r +1; "
     config.vm.provision :shell, :inline => pkg_cmd