From 0425f65e6373dc38cd18a6d0f2a50671544ad4b2 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 13 Jun 2013 17:53:38 -0700 Subject: [PATCH 1/6] Remove bsdtar by checking magic --- archive.go | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/archive.go b/archive.go index 06466627a1..5686408179 100644 --- a/archive.go +++ b/archive.go @@ -1,8 +1,10 @@ package docker import ( + "bytes" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "os" @@ -20,6 +22,37 @@ const ( Xz ) +func DetectCompression(source []byte) Compression { + for _, c := range source[:10] { + utils.Debugf("%x", c) + } + + sourceLen := len(source) + for compression, m := range map[Compression][]byte{ + Bzip2: {0x42, 0x5A, 0x68}, + Gzip: {0x1F, 0x8B, 0x08}, + Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, + } { + fail := false + if len(m) > sourceLen { + utils.Debugf("Len too short") + continue + } + i := 0 + for _, b := range m { + if b != source[i] { + fail = true + break + } + i++ + } + if !fail { + return compression + } + } + return Uncompressed +} + func (compression *Compression) Flag() string { switch *compression { case Bzip2: @@ -47,12 +80,21 @@ func (compression *Compression) Extension() string { } func Tar(path string, compression Compression) (io.Reader, error) { - cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".") - return CmdStream(cmd) + return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".")) } func Untar(archive io.Reader, path string) error { - cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x") + + buf := make([]byte, 10) + if _, err := archive.Read(buf); err != nil { + return err + } + compression := DetectCompression(buf) + archive = io.MultiReader(bytes.NewReader(buf), archive) + + utils.Debugf("Archive compression detected: %s", compression.Extension()) + + cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag()) cmd.Stdin = archive // Hardcode locale environment for predictable outcome regardless of host configuration. // (see https://github.com/dotcloud/docker/issues/355) From 6f7de49aa8771ef4bc35548f9aa07fee660f1844 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 14 Jun 2013 10:47:49 -0700 Subject: [PATCH 2/6] Add unit tests for tar/untar with multiple compression + detection --- archive_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/archive_test.go b/archive_test.go index f583604497..bb4235ad5b 100644 --- a/archive_test.go +++ b/archive_test.go @@ -1,10 +1,13 @@ package docker import ( + "bytes" + "fmt" "io" "io/ioutil" "os" "os/exec" + "path" "testing" "time" ) @@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) { } } -func TestTarUntar(t *testing.T) { - archive, err := Tar(".", Uncompressed) +func tarUntar(t *testing.T, origin string, compression Compression) error { + archive, err := Tar(origin, compression) if err != nil { t.Fatal(err) } + + buf := make([]byte, 10) + if _, err := archive.Read(buf); err != nil { + return err + } + archive = io.MultiReader(bytes.NewReader(buf), archive) + + detectedCompression := DetectCompression(buf) + if detectedCompression.Extension() != compression.Extension() { + return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension()) + } + tmp, err := ioutil.TempDir("", "docker-test-untar") if err != nil { - t.Fatal(err) + return err } defer os.RemoveAll(tmp) if err := Untar(archive, tmp); err != nil { - t.Fatal(err) + return err } if _, err := os.Stat(tmp); err != nil { - t.Fatalf("Error stating %s: %s", tmp, err.Error()) + return err + } + return nil +} + +func TestTarUntar(t *testing.T) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(origin) + if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { + t.Fatal(err) + } + + for _, c := range []Compression{ + Uncompressed, + Gzip, + Bzip2, + Xz, + } { + if err := tarUntar(t, origin, c); err != nil { + t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) + } } } From 79fe864d9a117193b4dce8b7fe156a55bbadcce6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 14 Jun 2013 10:58:16 -0700 Subject: [PATCH 3/6] Update docs --- README.md | 2 +- docs/sources/contributing/devenvironment.rst | 2 +- docs/sources/installation/binaries.rst | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c909e5431..d15ee3cc41 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ Setting up a dev environment Instructions that have been verified to work on Ubuntu 12.10, ```bash -sudo apt-get -y install lxc wget bsdtar curl golang git +sudo apt-get -y install lxc curl xz-utils golang git export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index 5d937c5a44..8b26688bcf 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -33,7 +33,7 @@ Installation sudo apt-get install python-software-properties sudo add-apt-repository ppa:gophers/go sudo apt-get update - sudo apt-get -y install lxc wget bsdtar curl golang-stable git + sudo apt-get -y install lxc xz-utils curl golang-stable git export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH diff --git a/docs/sources/installation/binaries.rst b/docs/sources/installation/binaries.rst index e7a07b6db1..6d87787752 100644 --- a/docs/sources/installation/binaries.rst +++ b/docs/sources/installation/binaries.rst @@ -30,8 +30,7 @@ Dependencies: * 3.8 Kernel (read more about :ref:`kernel`) * AUFS filesystem support * lxc -* bsdtar - +* xz-utils Get the docker binary: ---------------------- From 02c291d13be261f84cdfa1d51513bf4dba31ce72 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 17 Jun 2013 18:11:58 -0700 Subject: [PATCH 4/6] Fix bug on compression detection when chunck < 10bytes --- archive.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/archive.go b/archive.go index 5686408179..1e5b683765 100644 --- a/archive.go +++ b/archive.go @@ -1,7 +1,7 @@ package docker import ( - "bytes" + "bufio" "errors" "fmt" "github.com/dotcloud/docker/utils" @@ -85,17 +85,17 @@ func Tar(path string, compression Compression) (io.Reader, error) { func Untar(archive io.Reader, path string) error { - buf := make([]byte, 10) - if _, err := archive.Read(buf); err != nil { + bufferedArchive := bufio.NewReaderSize(archive, 10) + buf, err := bufferedArchive.Peek(10) + if err != nil { return err } compression := DetectCompression(buf) - archive = io.MultiReader(bytes.NewReader(buf), archive) utils.Debugf("Archive compression detected: %s", compression.Extension()) cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag()) - cmd.Stdin = archive + cmd.Stdin = bufferedArchive // Hardcode locale environment for predictable outcome regardless of host configuration. // (see https://github.com/dotcloud/docker/issues/355) cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"} From 6f511ac29b1c3cf1c2424b3c39ee14a32aecb5d7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 18 Jun 2013 10:23:45 -0700 Subject: [PATCH 5/6] Remove bsdtar dependency in various install scripts --- contrib/install.sh | 2 +- hack/Vagrantfile | 2 +- testing/Vagrantfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/install.sh b/contrib/install.sh index 7db577a9da..cf097da670 100755 --- a/contrib/install.sh +++ b/contrib/install.sh @@ -8,7 +8,7 @@ echo "Ensuring basic dependencies are installed..." apt-get -qq update -apt-get -qq install lxc wget bsdtar +apt-get -qq install lxc wget echo "Looking in /proc/filesystems to see if we have AUFS support..." if grep -q aufs /proc/filesystems diff --git a/hack/Vagrantfile b/hack/Vagrantfile index 318f835f4f..e02dfe06db 100644 --- a/hack/Vagrantfile +++ b/hack/Vagrantfile @@ -22,7 +22,7 @@ Vagrant::Config.run do |config| pkg_cmd = "touch #{DOCKER_PATH}; " # Install docker dependencies pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ - "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ + "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ "chown -R #{USER}.#{USER} #{GOPATH}; " \ "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" config.vm.provision :shell, :inline => pkg_cmd diff --git a/testing/Vagrantfile b/testing/Vagrantfile index e304a8d087..f2f6ca8248 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -30,7 +30,7 @@ Vagrant::Config.run do |config| # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ - "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; " + "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " # Activate new kernel pkg_cmd << "shutdown -r +1; " config.vm.provision :shell, :inline => pkg_cmd From 42ce68894a33a7d966f8dd767aee46dea6dbc346 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 18 Jun 2013 17:22:32 -0700 Subject: [PATCH 6/6] Fix issue within TestDelete. The archive is now consumed by graph functions --- graph_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/graph_test.go b/graph_test.go index 8dedb96669..18682338d9 100644 --- a/graph_test.go +++ b/graph_test.go @@ -192,11 +192,19 @@ func TestDelete(t *testing.T) { } assertNImages(graph, t, 0) + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } // Test 2 create (same name) / 1 delete img1, err := graph.Create(archive, nil, "Testing", "", nil) if err != nil { t.Fatal(err) } + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil { t.Fatal(err) } @@ -212,6 +220,10 @@ func TestDelete(t *testing.T) { } assertNImages(graph, t, 1) + archive, err = fakeTar() + if err != nil { + t.Fatal(err) + } // Test delete twice (pull -> rm -> pull -> rm) if err := graph.Register(archive, false, img1); err != nil { t.Fatal(err)