From 6d34c50e898507e461300ecf91ed661011bc15ab Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 26 Nov 2013 10:50:53 -0800 Subject: [PATCH 001/105] Increase max image depth to 127 --- graphdriver/aufs/aufs.go | 54 +++++++++++++++++-------- graphdriver/aufs/aufs_test.go | 71 +++++++++++++++++++++++++++++++++ graphdriver/aufs/mount_linux.go | 2 +- runtime.go | 6 ++- 4 files changed, 113 insertions(+), 20 deletions(-) diff --git a/graphdriver/aufs/aufs.go b/graphdriver/aufs/aufs.go index bd3ad6ebf6..558c64c5bb 100644 --- a/graphdriver/aufs/aufs.go +++ b/graphdriver/aufs/aufs.go @@ -26,11 +26,11 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/utils" - "log" "os" "os/exec" "path" "strings" + "syscall" ) func init() { @@ -313,24 +313,44 @@ func (a *Driver) Cleanup() error { return nil } -func (a *Driver) aufsMount(ro []string, rw, target string) error { - rwBranch := fmt.Sprintf("%v=rw", rw) - roBranches := "" - for _, layer := range ro { - roBranches += fmt.Sprintf("%v=ro+wh:", layer) - } - branches := fmt.Sprintf("br:%v:%v,xino=/dev/shm/aufs.xino", rwBranch, roBranches) +func (a *Driver) aufsMount(ro []string, rw, target string) (err error) { + defer func() { + if err != nil { + Unmount(target) + } + }() - //if error, try to load aufs kernel module - if err := mount("none", target, "aufs", 0, branches); err != nil { - log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...") - if err := exec.Command("modprobe", "aufs").Run(); err != nil { - return fmt.Errorf("Unable to load the AUFS module") + if err = a.tryMount(ro, rw, target); err != nil { + if err = a.mountRw(rw, target); err != nil { + return } - log.Printf("...module loaded.") - if err := mount("none", target, "aufs", 0, branches); err != nil { - return fmt.Errorf("Unable to mount using aufs %s", err) + + for _, layer := range ro { + branch := fmt.Sprintf("append:%s=ro+wh", layer) + if err = mount("none", target, "aufs", syscall.MS_REMOUNT, branch); err != nil { + return + } } } - return nil + return +} + +// Try to mount using the aufs fast path, if this fails then +// append ro layers. +func (a *Driver) tryMount(ro []string, rw, target string) (err error) { + var ( + rwBranch = fmt.Sprintf("%s=rw", rw) + roBranches = fmt.Sprintf("%s=ro+wh:", strings.Join(ro, "=ro+wh:")) + ) + return mount("none", target, "aufs", 0, fmt.Sprintf("br:%v:%v,xino=/dev/shm/aufs.xino", rwBranch, roBranches)) +} + +func (a *Driver) mountRw(rw, target string) error { + return mount("none", target, "aufs", 0, fmt.Sprintf("br:%s,xino=/dev/shm/aufs.xino", rw)) +} + +func rollbackMount(target string, err error) { + if err != nil { + Unmount(target) + } } diff --git a/graphdriver/aufs/aufs_test.go b/graphdriver/aufs/aufs_test.go index 2970311389..c43bd74038 100644 --- a/graphdriver/aufs/aufs_test.go +++ b/graphdriver/aufs/aufs_test.go @@ -1,7 +1,11 @@ package aufs import ( + "crypto/sha256" + "encoding/hex" + "fmt" "github.com/dotcloud/docker/archive" + "io/ioutil" "os" "path" "testing" @@ -621,3 +625,70 @@ func TestApplyDiff(t *testing.T) { t.Fatal(err) } } + +func hash(c string) string { + h := sha256.New() + fmt.Fprint(h, c) + return hex.EncodeToString(h.Sum(nil)) +} + +func TestMountMoreThan42Layers(t *testing.T) { + d := newDriver(t) + defer os.RemoveAll(tmp) + defer d.Cleanup() + var last string + var expected int + + for i := 1; i < 127; i++ { + expected++ + var ( + parent = fmt.Sprintf("%d", i-1) + current = fmt.Sprintf("%d", i) + ) + + if parent == "0" { + parent = "" + } else { + parent = hash(parent) + } + current = hash(current) + + if err := d.Create(current, parent); err != nil { + t.Logf("Current layer %d", i) + t.Fatal(err) + } + point, err := d.Get(current) + if err != nil { + t.Logf("Current layer %d", i) + t.Fatal(err) + } + f, err := os.Create(path.Join(point, current)) + if err != nil { + t.Logf("Current layer %d", i) + t.Fatal(err) + } + f.Close() + + if i%10 == 0 { + if err := os.Remove(path.Join(point, parent)); err != nil { + t.Logf("Current layer %d", i) + t.Fatal(err) + } + expected-- + } + last = current + } + + // Perform the actual mount for the top most image + point, err := d.Get(last) + if err != nil { + t.Fatal(err) + } + files, err := ioutil.ReadDir(point) + if err != nil { + t.Fatal(err) + } + if len(files) != expected { + t.Fatalf("Expected %d got %d", expected, len(files)) + } +} diff --git a/graphdriver/aufs/mount_linux.go b/graphdriver/aufs/mount_linux.go index 7122b73cea..8062bae420 100644 --- a/graphdriver/aufs/mount_linux.go +++ b/graphdriver/aufs/mount_linux.go @@ -2,6 +2,6 @@ package aufs import "syscall" -func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { +func mount(source string, target string, fstype string, flags uintptr, data string) error { return syscall.Mount(source, target, fstype, flags, data) } diff --git a/runtime.go b/runtime.go index f58be836bd..175bfa435e 100644 --- a/runtime.go +++ b/runtime.go @@ -24,8 +24,10 @@ import ( "time" ) -// Set the max depth to the aufs restriction -const MaxImageDepth = 42 +// Set the max depth to the aufs default that most +// kernels are compiled with +// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk +const MaxImageDepth = 127 var defaultDns = []string{"8.8.8.8", "8.8.4.4"} From 04c32495f69a48c34b753b66d7cdc4f2c695f405 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 5 Dec 2013 14:19:10 +1000 Subject: [PATCH 002/105] add a little prose to tell the user that run creates a container, and then starts it --- docs/sources/commandline/cli.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index d7bb03b954..43ef0f89b2 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -930,6 +930,9 @@ containers will not be deleted. -link="": Add link to another container (name:alias) -name="": Assign the specified name to the container. If no name is specific docker will generate a random name -P=false: Publish all exposed ports to the host interfaces + +``docker run`` ``creates`` a writeable container layer over the specified image, and then +``start``s it using the specified command (ie. is equivalent to the API ``/containers/create`` then ``/containers/(id)/start``). Known Issues (run -volumes-from) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 5a17c208cd4858a21d38b09e60ee8199880f02bb Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Thu, 5 Dec 2013 09:43:08 -0800 Subject: [PATCH 003/105] docs/installation/google: add enabling Google Compute Engine step --- docs/sources/installation/google.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/google.rst b/docs/sources/installation/google.rst index 1390c684bc..e3286d1d90 100644 --- a/docs/sources/installation/google.rst +++ b/docs/sources/installation/google.rst @@ -12,7 +12,7 @@ `Compute Engine `_ QuickStart for `Debian `_ ----------------------------------------------------------------------------------------------------------- -1. Go to `Google Cloud Console `_ and create a new Cloud Project with billing enabled. +1. Go to `Google Cloud Console `_ and create a new Cloud Project with `Compute Engine enabled `_. 2. Download and configure the `Google Cloud SDK `_ to use your project with the following commands: From f95f2789f274f3d98ea77cd6c10b899aa3db2dc6 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 2 Dec 2013 13:55:31 -0800 Subject: [PATCH 004/105] add docs for the new json format --- docs/sources/api/docker_remote_api.rst | 7 +++++++ docs/sources/api/docker_remote_api_v1.8.rst | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index f38a36ee1f..7cb7b323c1 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -55,6 +55,13 @@ What's new **New!** This endpoint now returns the host config for the container. +.. http:post:: /images/create +.. http:post:: /images/(name)/insert +.. http:post:: /images/(name)/push + + **New!** progressDetail object was added in the JSON. It's now possible + to get the current value and the total of the progress without having to + parse the string. v1.7 **** diff --git a/docs/sources/api/docker_remote_api_v1.8.rst b/docs/sources/api/docker_remote_api_v1.8.rst index 1e02817611..bf03194ae4 100644 --- a/docs/sources/api/docker_remote_api_v1.8.rst +++ b/docs/sources/api/docker_remote_api_v1.8.rst @@ -696,7 +696,7 @@ Create an image Content-Type: application/json {"status":"Pulling..."} - {"status":"Pulling", "progress":"1/? (n/a)"} + {"status":"Pulling", "progress":"1 B/ 100 B", "progressDetail":{"current":1, "total":100}} {"error":"Invalid..."} ... @@ -736,7 +736,7 @@ Insert a file in an image Content-Type: application/json {"status":"Inserting..."} - {"status":"Inserting", "progress":"1/? (n/a)"} + {"status":"Inserting", "progress":"1/? (n/a)", "progressDetail":{"current":1}} {"error":"Invalid..."} ... @@ -857,7 +857,7 @@ Push an image on the registry Content-Type: application/json {"status":"Pushing..."} - {"status":"Pushing", "progress":"1/? (n/a)"} + {"status":"Pushing", "progress":"1/? (n/a)", "progressDetail":{"current":1}}} {"error":"Invalid..."} ... From 03f8a3bbae5bfc0f43e1b3a2852d1557b21c5848 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 3 Dec 2013 17:48:19 -0500 Subject: [PATCH 005/105] Fixed #2136 - Added styles Added styling for versionadded, versionchanged, and deprecated. --- docs/theme/docker/static/css/main.css | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/theme/docker/static/css/main.css b/docs/theme/docker/static/css/main.css index 756adb1989..605dfc9774 100755 --- a/docs/theme/docker/static/css/main.css +++ b/docs/theme/docker/static/css/main.css @@ -410,3 +410,23 @@ dt:hover > a.headerlink { .admonition.seealso { border-color: #23cb1f; } + +.versionchanged, +.versionadded, +.versionmodified, +.deprecated { + font-size: larger; + font-weight: bold; +} + +.versionchanged { + color: lightseagreen; +} + +.versionadded { + color: mediumblue; +} + +.deprecated { + color: orangered; +} From f49eb29497620de597f4690a29957f44341f3673 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Thu, 5 Dec 2013 14:03:17 -0600 Subject: [PATCH 006/105] use mattdm/fedora in fedora doc and other cosmetic changes Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/fedora.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index 88b1037bfb..df3617c196 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -1,6 +1,6 @@ :title: Requirements and Installation on Fedora :description: Please note this project is currently under heavy development. It should not be used in production. -:keywords: Docker, Docker documentation, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux +:keywords: Docker, Docker documentation, fedora, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux .. _fedora: @@ -9,8 +9,6 @@ Fedora .. include:: install_header.inc -.. include:: install_unofficial.inc - Docker is available in **Fedora 19 and later**. Please note that due to the current Docker limitations Docker is able to run only on the **64 bit** architecture. @@ -24,19 +22,19 @@ Firstly, let's make sure our Fedora host is up-to-date. sudo yum -y upgrade -Next let's install the ``docker-io`` package which will install Docker on our host. +Next, let's install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash sudo yum -y install docker-io -Now it's installed lets start the Docker daemon. +Now that it's installed, let's start the Docker daemon. .. code-block:: bash sudo systemctl start docker -If we want Docker to start at boot we should also: +If we want Docker to start at boot, we should also: .. code-block:: bash @@ -46,7 +44,7 @@ Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t ubuntu /bin/bash + sudo docker run -i -t mattdm/fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. From 61fbf3d8e27541882ff7c446a7f0edfb4ed03d82 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Thu, 5 Dec 2013 14:21:03 -0600 Subject: [PATCH 007/105] yum upgrade on fedora not required before install Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/fedora.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index df3617c196..cbaa039fed 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -16,13 +16,7 @@ architecture. Installation ------------ -Firstly, let's make sure our Fedora host is up-to-date. - -.. code-block:: bash - - sudo yum -y upgrade - -Next, let's install the ``docker-io`` package which will install Docker on our host. +Install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash From 5d022f04453d3bac2e17b8622f013420778e8d5b Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Thu, 5 Dec 2013 16:08:08 -0600 Subject: [PATCH 008/105] add unofficial header back, yum update docker Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/fedora.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index cbaa039fed..67abda7184 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -9,6 +9,8 @@ Fedora .. include:: install_header.inc +.. include:: install_unofficial.inc + Docker is available in **Fedora 19 and later**. Please note that due to the current Docker limitations Docker is able to run only on the **64 bit** architecture. @@ -22,6 +24,9 @@ Install the ``docker-io`` package which will install Docker on our host. sudo yum -y install docker-io + # to keep docker-io up to date + sudo yum -y update docker-io + Now that it's installed, let's start the Docker daemon. .. code-block:: bash From 7ab4f37d6084c98e1f725be4cc88ea76bf952eb7 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Thu, 5 Dec 2013 16:11:33 -0600 Subject: [PATCH 009/105] separate block for yum update docker Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/fedora.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index 67abda7184..9525371f03 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -24,7 +24,10 @@ Install the ``docker-io`` package which will install Docker on our host. sudo yum -y install docker-io - # to keep docker-io up to date +To update the ``docker-io`` package + +.. code-block:: bash + sudo yum -y update docker-io Now that it's installed, let's start the Docker daemon. From eac95671f50a65279f8c39341c4f3be680add98a Mon Sep 17 00:00:00 2001 From: Pierre-Alain RIVIERE Date: Thu, 5 Dec 2013 19:02:19 +0100 Subject: [PATCH 010/105] refs #2490 : add a network page to docs --- docs/sources/use/index.rst | 1 + docs/sources/use/networking.rst | 147 ++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 docs/sources/use/networking.rst diff --git a/docs/sources/use/index.rst b/docs/sources/use/index.rst index 38f61ba744..7b61c203d5 100644 --- a/docs/sources/use/index.rst +++ b/docs/sources/use/index.rst @@ -18,6 +18,7 @@ Contents: baseimages port_redirection puppet + networking host_integration working_with_volumes working_with_links_names diff --git a/docs/sources/use/networking.rst b/docs/sources/use/networking.rst new file mode 100644 index 0000000000..c37035114e --- /dev/null +++ b/docs/sources/use/networking.rst @@ -0,0 +1,147 @@ +:title: Docker networking +:description: Docker networking +:keywords: network, networking, bridge, docker, documentation + + +Networking +========== + +Docker uses Linux bridge capabilities to provide network connectivity +to containers. The ``docker0`` bridge interface is managed by Docker itself +for this purpose. Thus, when the Docker daemon starts it : + +- creates the ``docker0`` bridge if not present +- searches for an IP address range which doesn't overlap with an existing route +- picks an IP in the selected range +- assigns this IP to the ``docker0`` bridge + + +.. code-block:: bash + + # List host bridges + $ sudo brctl show + bridge name bridge id STP enabled interfaces + docker0 8000.000000000000 no + + # Show docker0 IP address + $ sudo ifconfig docker0 + docker0 Link encap:Ethernet HWaddr xx:xx:xx:xx:xx:xx + inet addr:172.17.42.1 Bcast:0.0.0.0 Mask:255.255.0.0 + + + +At runtime, a :ref:`specific kind of virtual interface` is +given to each containers which is then bonded to the ``docker0`` bridge. +Each containers also receives a dedicated IP address from the same range +as ``docker0``. The ``docker0`` IP address is then used as the default +gateway for the containers. + +.. code-block:: bash + + # Run a container + $ sudo docker run -t -i -d base /bin/bash + 52f811c5d3d69edddefc75aff5a4525fc8ba8bcfa1818132f9dc7d4f7c7e78b4 + + $ sudo brctl show + bridge name bridge id STP enabled interfaces + docker0 8000.fef213db5a66 no vethQCDY1N + + +Above, ``docker0`` acts as a bridge for the ``vethQCDY1N`` interface which is dedicated +to the 52f811c5d3d6 container. + + +How to use a specific IP address range +--------------------------------------- +Docker will try hard to find an IP range which is not used by the host. +Even if it works for most cases, it's not bullet-proof and sometimes you need +to have more control over the IP addressing scheme. + +For this purpose, Docker allows you to manage the ``docker0`` bridge or +your own one using the ``-b=`` parameter. + +In this scenario: + +- ensure Docker is stopped +- create your own bridge (``bridge0`` for example) +- assign a specific IP to this bridge +- start Docker with the ``-b=bridge0`` parameter + + +.. code-block:: bash + + # Stop Docker + $ sudo service docker stop + + # Clean docker0 bridge and + # add your very own bridge0 + $ sudo ifconfig docker0 down + $ sudo brctl addbr bridge0 + $ sudo ifconfig bridge0 192.168.227.1 netmask 255.255.255.0 + + # Edit your Docker startup file + $ echo "DOCKER_OPTS=\"-b=bridge0\"" /etc/default/docker + + # Start Docker + $ sudo service docker start + + # Ensure bridge0 IP is not changed by Docker + $ sudo ifconfig bridge0 + bridge0 Link encap:Ethernet HWaddr xx:xx:xx:xx:xx:xx + inet addr:192.168.227.1 Bcast:192.168.227.255 Mask:255.255.255.0 + + # Run a container + $ docker run -i -t base /bin/bash + + # Container IP in the 192.168.227/24 range + root@261c272cd7d5:/# ifconfig eth0 + eth0 Link encap:Ethernet HWaddr xx:xx:xx:xx:xx:xx + inet addr:192.168.227.5 Bcast:192.168.227.255 Mask:255.255.255.0 + + # bridge0 IP as the default gateway + root@261c272cd7d5:/# route -n + Kernel IP routing table + Destination Gateway Genmask Flags Metric Ref Use Iface + 0.0.0.0 192.168.227.1 0.0.0.0 UG 0 0 0 eth0 + 192.168.227.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 + + # hits CTRL+P then CTRL+Q to detach + + # Display bridge info + $ sudo brctl show + bridge name bridge id STP enabled interfaces + bridge0 8000.fe7c2e0faebd no vethAQI2QT + + +Container intercommunication +------------------------------- +Containers can communicate with each other according to the ``icc`` parameter +value of the Docker daemon. + +- The default, ``-icc=true`` allows containers to communicate with each other. +- ``-icc=false`` means containers are isolated from each other. + +Under the hood, ``iptables`` is used by Docker to either accept or drop communication +between containers. + + +.. _vethxxxx-device: + +What's about the vethXXXX device? +----------------------------------- +Well. Things get complicated here. + +The ``vethXXXX`` interface is the host side of a point-to-point link between the +host and the corresponding container, the other side of the link being +materialized by the container's ``eth0`` interface. This pair (host ``vethXXX`` and +container ``eth0``) are connected like a tube. Everything that comes in one side will +come out the other side. + +All the plumbing is delegated to Linux network capabilities (check the ip link +command) and the namespaces infrastructure. + + +I want more +------------ +Jérôme Petazzoni has create ``pipework`` to connect together containers in +arbitrarily complex scenarios : https://github.com/jpetazzo/pipework From 4bea68dfa6a9c68232ff7a5c36c85fc827dae938 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Thu, 5 Dec 2013 17:16:31 -0800 Subject: [PATCH 011/105] Clean up quoting, wraps, and build error on code-block. --- docs/sources/commandline/cli.rst | 6 ++++-- docs/sources/use/ambassador_pattern_linking.rst | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index dd5339cce3..5cbd28c951 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -938,8 +938,10 @@ containers will not be deleted. -name="": Assign the specified name to the container. If no name is specific docker will generate a random name -P=false: Publish all exposed ports to the host interfaces -``docker run`` ``creates`` a writeable container layer over the specified image, and then -``start``s it using the specified command (ie. is equivalent to the API ``/containers/create`` then ``/containers/(id)/start``). +``'docker run'`` first ``'creates'`` a writeable container layer over +the specified image, and then ``'starts'`` it using the specified +command. That is, ``'docker run'`` is equivalent to the API +``/containers/create`` then ``/containers/(id)/start``. Known Issues (run -volumes-from) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/sources/use/ambassador_pattern_linking.rst b/docs/sources/use/ambassador_pattern_linking.rst index 98eb009d81..095324cb16 100644 --- a/docs/sources/use/ambassador_pattern_linking.rst +++ b/docs/sources/use/ambassador_pattern_linking.rst @@ -161,11 +161,12 @@ variable using the ``-e`` command line option. local ``1234`` port to the remote IP and port - in this case ``192.168.1.52:6379``. -.. code-block:: Dockerfile +:: # # - # first you need to build the docker-ut image using ./contrib/mkimage-unittest.sh + # first you need to build the docker-ut image + # using ./contrib/mkimage-unittest.sh # then # docker build -t SvenDowideit/ambassador . # docker tag SvenDowideit/ambassador ambassador From aea7418d8ac73f072a47affcaab04831c2f43bc2 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 5 Dec 2013 19:03:47 -0700 Subject: [PATCH 012/105] Revert "Add cgroup-bin dependency to our Ubuntu package" This reverts commit c81bb20f5b2b5d86059c6004e60ba23b03d30fe0. After re-reading the documentation: "The Recommends field should list packages that would be found together with this one in all but unusual installations." Thus, "Recommends" is an acceptable place for this dep, and anyone disabling that gets to keep the pieces. The main crux of why this needs to be reverted is because it breaks Debian completely because "lxc" and "cgroup-bin" can't be installed concurrently. --- hack/make/ubuntu | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/make/ubuntu b/hack/make/ubuntu index 258a869c24..578f254558 100644 --- a/hack/make/ubuntu +++ b/hack/make/ubuntu @@ -96,7 +96,6 @@ EOF --depends lxc \ --depends aufs-tools \ --depends iptables \ - --depends cgroup-bin \ --description "$PACKAGE_DESCRIPTION" \ --maintainer "$PACKAGE_MAINTAINER" \ --conflicts lxc-docker-virtual-package \ From 58b75f8f29b5fff388ec3d8d84249e122462c137 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Thu, 5 Dec 2013 18:08:56 -0800 Subject: [PATCH 013/105] Add debian mirrors. Fixes #3045. --- docs/sources/installation/ubuntulinux.rst | 27 ++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 93a1b28e2a..73e7c3a80e 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -63,7 +63,10 @@ Installation These instructions have changed for 0.6. If you are upgrading from an earlier version, you will need to follow them again. -Docker is available as a Debian package, which makes installation easy. +Docker is available as a Debian package, which makes installation +easy. **See the :ref:`installmirrors` section below if you are not in +the United States.** Other sources of the Debian packages may be +faster for you to install. First add the Docker repository key to your local keychain. You can use the ``apt-key`` command to check the fingerprint matches: ``36A1 D786 9245 C895 0F96 @@ -199,3 +202,25 @@ incoming connections on the Docker port (default 4243): sudo ufw allow 4243/tcp +.. _installmirrors: + +Mirrors +^^^^^^^ + +You should ``ping get.docker.io`` and compare the latency to the +following mirrors, and pick whichever one is best for you. + +Yandex +------ + +`Yandex `_ in Russia is mirroring the Docker Debian +packages, updating every 6 hours. Substitute +``http://mirror.yandex.ru/mirrors/docker/`` for +``http://get.docker.io/ubuntu`` in the instructions above. For example: + +.. code-block:: bash + + sudo sh -c "echo deb http://mirror.yandex.ru/mirrors/docker/ docker main\ + > /etc/apt/sources.list.d/docker.list" + sudo apt-get update + sudo apt-get install lxc-docker From a69bb25820df8951151c8c6cb5952ba5496463db Mon Sep 17 00:00:00 2001 From: Shawn Landden Date: Thu, 5 Dec 2013 23:43:54 -0800 Subject: [PATCH 014/105] specific kernel config --- docs/sources/installation/kernel.rst | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/sources/installation/kernel.rst b/docs/sources/installation/kernel.rst index 07ec09042f..b2cf4f1479 100644 --- a/docs/sources/installation/kernel.rst +++ b/docs/sources/installation/kernel.rst @@ -111,3 +111,40 @@ And replace it by the following one:: GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" Then run ``update-grub``, and reboot. + +Details +------- + +Networking: + +- CONFIG_BRIDGE +- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE +- CONFIG_NF_NAT +- CONFIG_NF_NAT_IPV4 +- CONFIG_NF_NAT_NEEDED + +LVM: + +- CONFIG_BLK_DEV_DM +- CONFIG_DM_THIN_PROVISIONING +- CONFIG_EXT4_FS + +Namespaces: + +- CONFIG_NAMESPACES +- CONFIG_UTS_NS +- CONFIG_IPC_NS +- CONFIG_UID_NS +- CONFIG_PID_NS +- CONFIG_NET_NS + +Cgroups: + +- CONFIG_CGROUPS + +Cgroup controllers (optional but highly recommended): + +- CONFIG_CGROUP_CPUACCT +- CONFIG_BLK_CGROUP +- CONFIG_MEMCG +- CONFIG_MEMCG_SWAP From 25d3db048e761ee2c61a2cc8d58196786616988b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 5 Dec 2013 16:22:49 -0800 Subject: [PATCH 015/105] Enable engine to take Stderr and Stdout for mocking in tests --- engine/engine.go | 13 ++++++++++--- integration/utils_test.go | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index 623af81bd1..34b28f64db 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,6 +3,7 @@ package engine import ( "fmt" "github.com/dotcloud/docker/utils" + "io" "log" "os" "runtime" @@ -34,6 +35,9 @@ type Engine struct { handlers map[string]Handler hack Hack // data for temporary hackery (see hack.go) id string + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader } func (eng *Engine) Root() string { @@ -82,6 +86,9 @@ func New(root string) (*Engine, error) { root: root, handlers: make(map[string]Handler), id: utils.RandomString(), + Stdout: os.Stdout, + Stderr: os.Stderr, + Stdin: os.Stdin, } // Copy existing global handlers for k, v := range globalHandlers { @@ -105,8 +112,8 @@ func (eng *Engine) Job(name string, args ...string) *Job { Stdout: NewOutput(), Stderr: NewOutput(), } - job.Stdout.Add(utils.NopWriteCloser(os.Stdout)) - job.Stderr.Add(utils.NopWriteCloser(os.Stderr)) + job.Stdout.Add(utils.NopWriteCloser(eng.Stdout)) + job.Stderr.Add(utils.NopWriteCloser(eng.Stderr)) handler, exists := eng.handlers[name] if exists { job.handler = handler @@ -116,5 +123,5 @@ func (eng *Engine) Job(name string, args ...string) *Job { func (eng *Engine) Logf(format string, args ...interface{}) (n int, err error) { prefixedFormat := fmt.Sprintf("[%s] %s\n", eng, strings.TrimRight(format, "\n")) - return fmt.Fprintf(os.Stderr, prefixedFormat, args...) + return fmt.Fprintf(eng.Stderr, prefixedFormat, args...) } diff --git a/integration/utils_test.go b/integration/utils_test.go index 2feaf25396..62ec5a51fd 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -185,6 +185,8 @@ func NewTestEngine(t utils.Fataler) *engine.Engine { if err != nil { t.Fatal(err) } + eng.Stdout = ioutil.Discard + eng.Stderr = ioutil.Discard // Load default plugins // (This is manually copied and modified from main() until we have a more generic plugin system) job := eng.Job("initapi") From 02ef8ec3ca1a9b16b905b671ade74cf24d11bb3a Mon Sep 17 00:00:00 2001 From: tang0th Date: Fri, 6 Dec 2013 15:47:24 +0000 Subject: [PATCH 016/105] Update archlinux.rst as packages have changed The docker package has been added into the Arch Linux community repo, this means that the package names and installation instructions have slightly changed. --- docs/sources/installation/archlinux.rst | 33 ++++++++++++++++--------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/sources/installation/archlinux.rst b/docs/sources/installation/archlinux.rst index ae56badf4c..44249b178f 100644 --- a/docs/sources/installation/archlinux.rst +++ b/docs/sources/installation/archlinux.rst @@ -12,21 +12,22 @@ Arch Linux .. include:: install_unofficial.inc Installing on Arch Linux is not officially supported but can be handled via -one of the following AUR packages: +the package in community: -* `lxc-docker `_ -* `lxc-docker-git `_ -* `lxc-docker-nightly `_ +* `docker `_ -The lxc-docker package will install the latest tagged version of docker. -The lxc-docker-git package will build from the current master branch. -The lxc-docker-nightly package will install the latest build. +or the following AUR package: + +* `docker-git `_ + +The docker package will install the latest tagged version of docker. +The docker-git package will build from the current master branch. Dependencies ------------ Docker depends on several packages which are specified as dependencies in -the AUR packages. The core dependencies are: +the packages. The core dependencies are: * bridge-utils * device-mapper @@ -37,15 +38,23 @@ the AUR packages. The core dependencies are: Installation ------------ +For the normal package a simple +:: + + pacman -S docker + +is all that is needed. + +For the AUR package execute: +:: + + yaourt -S lxc-docker-git + The instructions here assume **yaourt** is installed. See `Arch User Repository `_ for information on building and installing packages from the AUR if you have not done so before. -:: - - yaourt -S lxc-docker - Starting Docker --------------- From a015f38f4a886518828fe3807ee7dc9ff8ab9585 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 6 Dec 2013 10:13:07 -0600 Subject: [PATCH 017/105] devmapper: add missing defines Add some missing defines which are needed for compiling on older systems like RHEL 6. --- graphdriver/devmapper/devmapper_wrapper.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 9a36e66225..80d430e2bf 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -8,6 +8,14 @@ package devmapper #include // FIXME: present only for defines, maybe we can remove it? #include // FIXME: present only for BLKGETSIZE64, maybe we can remove it? +#ifndef LOOP_CTL_GET_FREE + #define LOOP_CTL_GET_FREE 0x4C82 +#endif + +#ifndef LO_FLAGS_PARTSCAN + #define LO_FLAGS_PARTSCAN 8 +#endif + // FIXME: Can't we find a way to do the logging in pure Go? extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str); @@ -55,7 +63,6 @@ type ( } ) -// FIXME: Make sure the values are defined in C // IOCTL consts const ( BlkGetSize64 = C.BLKGETSIZE64 From 6720bfb24312a558a27453ce104beeb04c53ec68 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 5 Dec 2013 15:10:44 -0800 Subject: [PATCH 018/105] Adjusted handling of inactive user login The return status for inactive users was being checked too early in the process, so I moved it from just after the handling of POST /v1/users/ to after getting the response from GET /v1/users/ --- auth/auth.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 85844a5472..e88fb908f9 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -192,13 +192,6 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e } else { status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." } - } else if reqStatusCode == 403 { - if loginAgainstOfficialIndex { - return "", fmt.Errorf("Login: Your account hasn't been activated. " + - "Please check your e-mail for a confirmation link.") - } - return "", fmt.Errorf("Login: Your account hasn't been activated. " + - "Please see the documentation of the registry " + serverAddress + " for instructions how to activate it.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { req, err := factory.NewRequest("GET", serverAddress+"users/", nil) @@ -216,9 +209,13 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e status = "Login Succeeded" } else if resp.StatusCode == 401 { return "", fmt.Errorf("Wrong login/password, please try again") + } else if resp.StatusCode == 403 { + if loginAgainstOfficialIndex { + return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") + } + return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, - resp.StatusCode, resp.Header) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } } else { return "", fmt.Errorf("Registration: %s", reqBody) @@ -236,7 +233,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err - } + } if resp.StatusCode == 200 { status = "Login Succeeded" } else if resp.StatusCode == 401 { From 13da09d22bcae553cb11b5f906d3c260db571b7d Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 6 Dec 2013 12:52:32 +1000 Subject: [PATCH 019/105] change the policy wrt $ sudo docker to simplify auto-testing --- docs/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index e8e75c128c..3f4725a583 100644 --- a/docs/README.md +++ b/docs/README.md @@ -51,15 +51,15 @@ directory: ###Alternative Installation: Docker Container If you're running ``docker`` on your development machine then you may -find it easier and cleaner to use the Dockerfile. This installs Sphinx +find it easier and cleaner to use the docs Dockerfile. This installs Sphinx in a container, adds the local ``docs/`` directory and builds the HTML docs inside the container, even starting a simple HTTP server on port -8000 so that you can connect and see your changes. Just run ``docker -build .`` and run the resulting image. This is the equivalent to -``make clean server`` since each container starts clean. +8000 so that you can connect and see your changes. -In the ``docs/`` directory, run: - ```docker build -t docker:docs . && docker run -p 8000:8000 docker:docs``` +In the ``docker`` source directory, run: + ```make doc``` + +This is the equivalent to ``make clean server`` since each container starts clean. Usage ----- @@ -128,7 +128,8 @@ Guides on using sphinx * Code examples - * Start without $, so it's easy to copy and paste. + * Start typed commands with ``$ `` (dollar space) so that they + are easily differentiated from program output. * Use "sudo" with docker to ensure that your command is runnable even if they haven't [used the *docker* group](http://docs.docker.io/en/latest/use/basics/#why-sudo). From 05f416d869777aa613a5a197ad287c18b77366d3 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 6 Dec 2013 11:55:56 -0800 Subject: [PATCH 020/105] fix jsonmessage in build --- buildfile.go | 4 ++-- commands.go | 2 -- utils/jsonmessage.go | 7 +++++-- utils/streamformatter.go | 12 ++++++++++++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/buildfile.go b/buildfile.go index 170d739ee4..adbf98d274 100644 --- a/buildfile.go +++ b/buildfile.go @@ -375,7 +375,7 @@ type StdoutFormater struct { } func (sf *StdoutFormater) Write(buf []byte) (int, error) { - formattedBuf := sf.StreamFormatter.FormatStatus("", "%s", string(buf)) + formattedBuf := sf.StreamFormatter.FormatStream(string(buf)) n, err := sf.Writer.Write(formattedBuf) if n != len(formattedBuf) { return n, io.ErrShortWrite @@ -389,7 +389,7 @@ type StderrFormater struct { } func (sf *StderrFormater) Write(buf []byte) (int, error) { - formattedBuf := sf.StreamFormatter.FormatStatus("", "%s", "\033[91m"+string(buf)+"\033[0m") + formattedBuf := sf.StreamFormatter.FormatStream("\033[91m" + string(buf) + "\033[0m") n, err := sf.Writer.Write(formattedBuf) if n != len(formattedBuf) { return n, io.ErrShortWrite diff --git a/commands.go b/commands.go index b7e7c08cac..48c0241b17 100644 --- a/commands.go +++ b/commands.go @@ -229,8 +229,6 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if context != nil { headers.Set("Content-Type", "application/tar") } - // Temporary hack to fix displayJSON behavior - cli.isTerminal = false err = cli.stream("POST", fmt.Sprintf("/build?%s", v.Encode()), body, cli.out, headers) if jerr, ok := err.(*utils.JSONError); ok { return &utils.StatusError{Status: jerr.Message, StatusCode: jerr.Code} diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index c36ec27763..82fe631b49 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -66,6 +66,7 @@ func (p *JSONProgress) String() string { } type JSONMessage struct { + Stream string `json:"stream,omitempty"` Status string `json:"status,omitempty"` Progress *JSONProgress `json:"progressDetail,omitempty"` ProgressMessage string `json:"progress,omitempty"` //deprecated @@ -87,7 +88,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { if isTerminal { // [2K = erase entire current line fmt.Fprintf(out, "%c[2K\r", 27) - endl = "\r\n" + endl = "\r" } if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) @@ -102,8 +103,10 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { fmt.Fprintf(out, "%s %s%s", jm.Status, jm.Progress.String(), endl) } else if jm.ProgressMessage != "" { //deprecated fmt.Fprintf(out, "%s %s%s", jm.Status, jm.ProgressMessage, endl) + } else if jm.Stream != "" { + fmt.Fprintf(out, "%s%s", jm.Stream, endl) } else { - fmt.Fprintf(out, "%s%s", jm.Status, endl) + fmt.Fprintf(out, "%s%s\n", jm.Status, endl) } return nil } diff --git a/utils/streamformatter.go b/utils/streamformatter.go index 60863d2aa6..0c41d0bddd 100644 --- a/utils/streamformatter.go +++ b/utils/streamformatter.go @@ -14,6 +14,18 @@ func NewStreamFormatter(json bool) *StreamFormatter { return &StreamFormatter{json, false} } +func (sf *StreamFormatter) FormatStream(str string) []byte { + sf.used = true + if sf.json { + b, err := json.Marshal(&JSONMessage{Stream: str}) + if err != nil { + return sf.FormatError(err) + } + return b + } + return []byte(str + "\r") +} + func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { sf.used = true str := fmt.Sprintf(format, a...) From becb13dc26e8b40be4eec2da933705b2a378db35 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 6 Dec 2013 12:02:21 -0800 Subject: [PATCH 021/105] update doc --- docs/sources/api/docker_remote_api_v1.8.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/api/docker_remote_api_v1.8.rst b/docs/sources/api/docker_remote_api_v1.8.rst index bf03194ae4..909e9a7a7f 100644 --- a/docs/sources/api/docker_remote_api_v1.8.rst +++ b/docs/sources/api/docker_remote_api_v1.8.rst @@ -1008,8 +1008,8 @@ Build an image from Dockerfile via stdin HTTP/1.1 200 OK Content-Type: application/json - {"status":"Step 1..."} - {"status":"..."} + {"stream":"Step 1..."} + {"stream":"..."} {"error":"Error...", "errorDetail":{"code": 123, "message": "Error..."}} From 2c646b2d466b1401fccbfd453a68f0023dd5e300 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 6 Dec 2013 13:56:09 -0800 Subject: [PATCH 022/105] disable progressbar in non-terminal --- utils/jsonmessage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index 82fe631b49..cc467162f9 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -89,6 +89,8 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { // [2K = erase entire current line fmt.Fprintf(out, "%c[2K\r", 27) endl = "\r" + } else if jm.Progress != nil { //disable progressbar in non-terminal + return nil } if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) From 228091c79ed043d1896afb04cd0573ee972c2607 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 6 Dec 2013 14:27:10 -0800 Subject: [PATCH 023/105] added authConfig to docker build --- api.go | 13 ++++++++++++- buildfile.go | 8 ++++++-- commands.go | 6 ++++++ docs/sources/api/docker_remote_api_v1.8.rst | 1 + 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 0aa9695702..fb59f5efd3 100644 --- a/api.go +++ b/api.go @@ -912,6 +912,17 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ rawRm := r.FormValue("rm") repoName, tag := utils.ParseRepositoryTag(repoName) + authEncoded := r.Header.Get("X-Registry-Auth") + authConfig := &auth.AuthConfig{} + if authEncoded != "" { + authJson := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authEncoded)) + if err := json.NewDecoder(authJson).Decode(authConfig); err != nil { + // for a pull it is not an error if no auth was given + // to increase compatibility with the existing api it is defaulting to be empty + authConfig = &auth.AuthConfig{} + } + } + var context io.Reader if remoteURL == "" { @@ -978,7 +989,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ Writer: utils.NewWriteFlusher(w), StreamFormatter: sf, }, - !suppressOutput, !noCache, rm, utils.NewWriteFlusher(w), sf) + !suppressOutput, !noCache, rm, utils.NewWriteFlusher(w), sf, authConfig) id, err := b.Build(context) if err != nil { if sf.Used() { diff --git a/buildfile.go b/buildfile.go index 170d739ee4..be4c56c96d 100644 --- a/buildfile.go +++ b/buildfile.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -33,6 +34,8 @@ type buildFile struct { utilizeCache bool rm bool + authConfig *auth.AuthConfig + tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -57,7 +60,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.outOld, b.sf, nil, nil, true); err != nil { + if err := b.srv.ImagePull(remote, tag, b.outOld, b.sf, b.authConfig, nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) @@ -568,7 +571,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occurred during the build\n") } -func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeCache, rm bool, outOld io.Writer, sf *utils.StreamFormatter) BuildFile { +func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeCache, rm bool, outOld io.Writer, sf *utils.StreamFormatter, auth *auth.AuthConfig) BuildFile { return &buildFile{ runtime: srv.runtime, srv: srv, @@ -581,6 +584,7 @@ func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeC utilizeCache: utilizeCache, rm: rm, sf: sf, + authConfig: auth, outOld: outOld, } } diff --git a/commands.go b/commands.go index b7e7c08cac..4ca69d26d2 100644 --- a/commands.go +++ b/commands.go @@ -226,6 +226,12 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } headers := http.Header(make(map[string][]string)) + buf, err := json.Marshal(cli.configFile) + if err != nil { + return err + } + headers.Add("X-Registry-Auth", base64.URLEncoding.EncodeToString(buf)) + if context != nil { headers.Set("Content-Type", "application/tar") } diff --git a/docs/sources/api/docker_remote_api_v1.8.rst b/docs/sources/api/docker_remote_api_v1.8.rst index bf03194ae4..b5eb35215a 100644 --- a/docs/sources/api/docker_remote_api_v1.8.rst +++ b/docs/sources/api/docker_remote_api_v1.8.rst @@ -1026,6 +1026,7 @@ Build an image from Dockerfile via stdin :query q: suppress verbose build output :query nocache: do not use the cache when building the image :reqheader Content-type: should be set to ``"application/tar"``. + :reqheader X-Registry-Auth: base64-encoded AuthConfig object :statuscode 200: no error :statuscode 500: server error From fdb3de7b11e4b85ec86f76aad51290730c037668 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 17 Nov 2013 00:26:04 +0000 Subject: [PATCH 024/105] Engine: new command 'kill' sends a signal to a running container --- api.go | 16 ++++------------ server.go | 29 +++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/api.go b/api.go index 0aa9695702..61890d86d4 100644 --- a/api.go +++ b/api.go @@ -150,19 +150,11 @@ func postContainersKill(srv *Server, version float64, w http.ResponseWriter, r * if err := parseForm(r); err != nil { return err } - name := vars["name"] - - signal := 0 - if r != nil { - if s := r.Form.Get("signal"); s != "" { - s, err := strconv.Atoi(s) - if err != nil { - return err - } - signal = s - } + job := srv.Eng.Job("kill", vars["name"]) + if sig := r.Form.Get("signal"); sig != "" { + job.Args = append(job.Args, sig) } - if err := srv.ContainerKill(name, signal); err != nil { + if err := job.Run(); err != nil { return err } w.WriteHeader(http.StatusNoContent) diff --git a/server.go b/server.go index 49b25f7098..297ac33549 100644 --- a/server.go +++ b/server.go @@ -23,6 +23,7 @@ import ( "path/filepath" "runtime" "strings" + "strconv" "sync" "syscall" "time" @@ -78,6 +79,9 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("kill", srv.ContainerKill); err != nil { + return err.Error() + } if err := job.Eng.Register("serveapi", srv.ListenAndServe); err != nil { job.Error(err) return engine.StatusErr @@ -169,25 +173,38 @@ func (srv *Server) versionInfos() []utils.VersionInfo { // If no signal is given (sig 0), then Kill with SIGKILL and wait // for the container to exit. // If a signal is given, then just send it to the container and return. -func (srv *Server) ContainerKill(name string, sig int) error { +func (srv *Server) ContainerKill(job *engine.Job) string { + if n := len(job.Args); n < 1 || n > 2 { + return fmt.Sprintf("Usage: %s CONTAINER [SIGNAL]", job.Name) + } + name := job.Args[0] + var sig uint64 + if len(job.Args) == 2 && job.Args[1] != "" { + var err error + // The largest legal signal is 31, so let's parse on 5 bits + sig, err = strconv.ParseUint(job.Args[1], 10, 5) + if err != nil { + return fmt.Sprintf("Invalid signal: %s", job.Args[1]) + } + } if container := srv.runtime.Get(name); container != nil { // If no signal is passed, perform regular Kill (SIGKILL + wait()) if sig == 0 { if err := container.Kill(); err != nil { - return fmt.Errorf("Cannot kill container %s: %s", name, err) + return fmt.Sprintf("Cannot kill container %s: %s", name, err) } srv.LogEvent("kill", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { // Otherwise, just send the requested signal - if err := container.kill(sig); err != nil { - return fmt.Errorf("Cannot kill container %s: %s", name, err) + if err := container.kill(int(sig)); err != nil { + return fmt.Sprintf("Cannot kill container %s: %s", name, err) } // FIXME: Add event for signals } } else { - return fmt.Errorf("No such container: %s", name) + return fmt.Sprintf("No such container: %s", name) } - return nil + return "0" } func (srv *Server) ContainerExport(name string, out io.Writer) error { From 2546a2c645c59d094bc2c09dc9eb0d8eb5c36786 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 17 Nov 2013 00:26:42 +0000 Subject: [PATCH 025/105] Hack: use new 'kill' command in integration tests --- integration/server_test.go | 2 +- integration/utils_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/server_test.go b/integration/server_test.go index 3e85effe8f..9ad8accf26 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -199,7 +199,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerKill(id, 0); err != nil { + if err := eng.Job("kill", id).Run(); err != nil { t.Fatal(err) } diff --git a/integration/utils_test.go b/integration/utils_test.go index 62ec5a51fd..8ee1a0af08 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -105,7 +105,7 @@ func containerWaitTimeout(eng *engine.Engine, id string, t utils.Fataler) error } func containerKill(eng *engine.Engine, id string, t utils.Fataler) { - if err := getContainer(eng, id, t).Kill(); err != nil { + if err := eng.Job("kill", id).Run(); err != nil { t.Fatal(err) } } From 9b1930c5a07fe3f696ec54f5d429b8020cc9086f Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 18 Nov 2013 20:17:51 +0000 Subject: [PATCH 026/105] gofmt --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index 297ac33549..2117c7bfbb 100644 --- a/server.go +++ b/server.go @@ -22,8 +22,8 @@ import ( "path" "path/filepath" "runtime" - "strings" "strconv" + "strings" "sync" "syscall" "time" From 427bdb60e73ad059e813babd981168ea2464ea1f Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Nov 2013 03:41:17 +0000 Subject: [PATCH 027/105] Engine: port 'kill' to the new integer status. --- engine/streams.go | 2 +- server.go | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/streams.go b/engine/streams.go index ff7049074a..697407f1f4 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -89,7 +89,7 @@ func (o *Output) Write(p []byte) (n int, err error) { firstErr = err } } - return len(p), firstErr + return len(p), err } // Close unregisters all destinations and waits for all background diff --git a/server.go b/server.go index 2117c7bfbb..eba1e11b7f 100644 --- a/server.go +++ b/server.go @@ -80,7 +80,8 @@ func jobInitApi(job *engine.Job) engine.Status { return engine.StatusErr } if err := job.Eng.Register("kill", srv.ContainerKill); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } if err := job.Eng.Register("serveapi", srv.ListenAndServe); err != nil { job.Error(err) @@ -173,9 +174,10 @@ func (srv *Server) versionInfos() []utils.VersionInfo { // If no signal is given (sig 0), then Kill with SIGKILL and wait // for the container to exit. // If a signal is given, then just send it to the container and return. -func (srv *Server) ContainerKill(job *engine.Job) string { +func (srv *Server) ContainerKill(job *engine.Job) engine.Status { if n := len(job.Args); n < 1 || n > 2 { - return fmt.Sprintf("Usage: %s CONTAINER [SIGNAL]", job.Name) + job.Errorf("Usage: %s CONTAINER [SIGNAL]", job.Name) + return engine.StatusErr } name := job.Args[0] var sig uint64 @@ -184,27 +186,31 @@ func (srv *Server) ContainerKill(job *engine.Job) string { // The largest legal signal is 31, so let's parse on 5 bits sig, err = strconv.ParseUint(job.Args[1], 10, 5) if err != nil { - return fmt.Sprintf("Invalid signal: %s", job.Args[1]) + job.Errorf("Invalid signal: %s", job.Args[1]) + return engine.StatusErr } } if container := srv.runtime.Get(name); container != nil { // If no signal is passed, perform regular Kill (SIGKILL + wait()) if sig == 0 { if err := container.Kill(); err != nil { - return fmt.Sprintf("Cannot kill container %s: %s", name, err) + job.Errorf("Cannot kill container %s: %s", name, err) + return engine.StatusErr } srv.LogEvent("kill", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { // Otherwise, just send the requested signal if err := container.kill(int(sig)); err != nil { - return fmt.Sprintf("Cannot kill container %s: %s", name, err) + job.Errorf("Cannot kill container %s: %s", name, err) + return engine.StatusErr } // FIXME: Add event for signals } } else { - return fmt.Sprintf("No such container: %s", name) + job.Errorf("No such container: %s", name) + return engine.StatusErr } - return "0" + return engine.StatusOK } func (srv *Server) ContainerExport(name string, out io.Writer) error { From 3569d080afc721a1eb04ad4a65f1309b53da1238 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 25 Nov 2013 01:05:59 +0000 Subject: [PATCH 028/105] New engine command: 'wait' --- api.go | 14 +++++++++----- integration/server_test.go | 4 ++-- server.go | 20 ++++++++++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index 0aa9695702..aebcd63533 100644 --- a/api.go +++ b/api.go @@ -710,14 +710,18 @@ func postContainersWait(srv *Server, version float64, w http.ResponseWriter, r * if vars == nil { return fmt.Errorf("Missing parameter") } - name := vars["name"] - - status, err := srv.ContainerWait(name) + job := srv.Eng.Job("wait", vars["name"]) + var statusStr string + job.Stdout.AddString(&statusStr) + if err := job.Run(); err != nil { + return err + } + // Parse a 16-bit encoded integer to map typical unix exit status. + status, err := strconv.ParseInt(statusStr, 10, 16) if err != nil { return err } - - return writeJSON(w, http.StatusOK, &APIWait{StatusCode: status}) + return writeJSON(w, http.StatusOK, &APIWait{StatusCode: int(status)}) } func postContainersResize(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/integration/server_test.go b/integration/server_test.go index 3e85effe8f..2f108c2b78 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -256,7 +256,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - if _, err := srv.ContainerWait(containerID); err != nil { + if err := eng.Job("wait", containerID).Run(); err != nil { t.Fatal(err) } @@ -281,7 +281,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - if _, err := srv.ContainerWait(containerID); err != nil { + if err := eng.Job("wait", containerID).Run(); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index 49b25f7098..640b4c44aa 100644 --- a/server.go +++ b/server.go @@ -82,6 +82,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("wait", srv.ContainerWait); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -1718,11 +1722,19 @@ func (srv *Server) ContainerStop(name string, t int) error { return nil } -func (srv *Server) ContainerWait(name string) (int, error) { - if container := srv.runtime.Get(name); container != nil { - return container.Wait(), nil +func (srv *Server) ContainerWait(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + job.Errorf("Usage: %s", job.Name) + return engine.StatusErr } - return 0, fmt.Errorf("No such container: %s", name) + name := job.Args[0] + if container := srv.runtime.Get(name); container != nil { + status := container.Wait() + job.Printf("%d\n", status) + return engine.StatusOK + } + job.Errorf("%s: no such container: %s", job.Name, name) + return engine.StatusErr } func (srv *Server) ContainerResize(name string, h, w int) error { From 3c1f3be032be4cd4a8c9235ed35e3a269e5f524e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 6 Dec 2013 17:31:09 -0800 Subject: [PATCH 029/105] Update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 39e898a4f9..c0ab427224 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +0.7.1-dev From 50f3a696bdfde5aed8f020fbac24af4c6c653ca9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 01:33:05 +0000 Subject: [PATCH 030/105] Engine: don't log job stdout to engine stdout (it might be non-text output, for example tar data for 'export' --- engine/engine.go | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/engine.go b/engine/engine.go index 34b28f64db..6042a649f7 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -112,7 +112,6 @@ func (eng *Engine) Job(name string, args ...string) *Job { Stdout: NewOutput(), Stderr: NewOutput(), } - job.Stdout.Add(utils.NopWriteCloser(eng.Stdout)) job.Stderr.Add(utils.NopWriteCloser(eng.Stderr)) handler, exists := eng.handlers[name] if exists { From 9656cdf0c248fda629fe9f13cf4ce4d3fabd35a4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 01:33:37 +0000 Subject: [PATCH 031/105] Engine: 'export' returns a raw archive of a container's filesystem --- api.go | 9 +++++---- server.go | 26 +++++++++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index 0aa9695702..0bdb928f73 100644 --- a/api.go +++ b/api.go @@ -173,10 +173,11 @@ func getContainersExport(srv *Server, version float64, w http.ResponseWriter, r if vars == nil { return fmt.Errorf("Missing parameter") } - name := vars["name"] - - if err := srv.ContainerExport(name, w); err != nil { - utils.Errorf("%s", err) + job := srv.Eng.Job("export", vars["name"]) + if err := job.Stdout.Add(w); err != nil { + return err + } + if err := job.Run(); err != nil { return err } return nil diff --git a/server.go b/server.go index 49b25f7098..314d98d525 100644 --- a/server.go +++ b/server.go @@ -70,6 +70,10 @@ func jobInitApi(job *engine.Job) engine.Status { if srv.runtime.networkManager.bridgeNetwork != nil { job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP) } + if err := job.Eng.Register("export", srv.ContainerExport); err != nil { + job.Error(err) + return engine.StatusErr + } if err := job.Eng.Register("create", srv.ContainerCreate); err != nil { job.Error(err) return engine.StatusErr @@ -190,22 +194,30 @@ func (srv *Server) ContainerKill(name string, sig int) error { return nil } -func (srv *Server) ContainerExport(name string, out io.Writer) error { +func (srv *Server) ContainerExport(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + job.Errorf("Usage: %s container_id", job.Name) + return engine.StatusErr + } + name := job.Args[0] if container := srv.runtime.Get(name); container != nil { - data, err := container.Export() if err != nil { - return err + job.Errorf("%s: %s", name, err) + return engine.StatusErr } // Stream the entire contents of the container (basically a volatile snapshot) - if _, err := io.Copy(out, data); err != nil { - return err + if _, err := io.Copy(job.Stdout, data); err != nil { + job.Errorf("%s: %s", name, err) + return engine.StatusErr } + // FIXME: factor job-specific LogEvent to engine.Job.Run() srv.LogEvent("export", container.ID, srv.runtime.repositories.ImageName(container.Image)) - return nil + return engine.StatusOK } - return fmt.Errorf("No such container: %s", name) + job.Errorf("No such container: %s", name) + return engine.StatusErr } // ImageExport exports all images with the given tag. All versions From edace08327675967700a5ff8093ee3104a91772f Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 01:47:03 +0000 Subject: [PATCH 032/105] Hack: stats.sh prints useful project stats for maintainers --- hack/stats.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 hack/stats.sh diff --git a/hack/stats.sh b/hack/stats.sh new file mode 100755 index 0000000000..2053e583a2 --- /dev/null +++ b/hack/stats.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +## Run this script from the root of the docker repository +## to query project stats useful to the maintainers. +## You will need to install `pulls` and `issues` from +## http://github.com/crosbymichael/pulls + +set -e + +echo -n "Open pulls: " +PULLS=$(pulls | wc -l); let PULLS=$PULLS-1 +echo $PULLS + +echo -n "Pulls alru: " +pulls alru + +echo -n "Open issues: " +ISSUES=$(issues list | wc -l); let ISSUES=$ISSUES-1 +echo $ISSUES + +echo -n "Issues alru: " +issues alru From a80c059baeff0b7d563499e963059a183c7e1bd9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 06:06:05 +0000 Subject: [PATCH 033/105] Engine: break out Env utilities into their own type - Env --- engine/engine.go | 1 + engine/env.go | 221 +++++++++++++++++++++++++++++++++++++++++++++++ engine/job.go | 140 ++++-------------------------- 3 files changed, 237 insertions(+), 125 deletions(-) create mode 100644 engine/env.go diff --git a/engine/engine.go b/engine/engine.go index 34b28f64db..44facf5277 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -111,6 +111,7 @@ func (eng *Engine) Job(name string, args ...string) *Job { Stdin: NewInput(), Stdout: NewOutput(), Stderr: NewOutput(), + env: &Env{}, } job.Stdout.Add(utils.NopWriteCloser(eng.Stdout)) job.Stderr.Add(utils.NopWriteCloser(eng.Stderr)) diff --git a/engine/env.go b/engine/env.go new file mode 100644 index 0000000000..5e4f59395a --- /dev/null +++ b/engine/env.go @@ -0,0 +1,221 @@ +package engine + +import ( + "strings" + "io" + "encoding/json" + "strconv" + "bytes" + "fmt" +) + +type Env []string + + +func (env *Env) Get(key string) (value string) { + // FIXME: use Map() + for _, kv := range *env { + if strings.Index(kv, "=") == -1 { + continue + } + parts := strings.SplitN(kv, "=", 2) + if parts[0] != key { + continue + } + if len(parts) < 2 { + value = "" + } else { + value = parts[1] + } + } + return +} + +func (env *Env) Exists(key string) bool { + _, exists := env.Map()[key] + return exists +} + +func (env *Env) GetBool(key string) (value bool) { + s := strings.ToLower(strings.Trim(env.Get(key), " \t")) + if s == "" || s == "0" || s == "no" || s == "false" || s == "none" { + return false + } + return true +} + + +func (env *Env) SetBool(key string, value bool) { + if value { + env.Set(key, "1") + } else { + env.Set(key, "0") + } +} + +func (env *Env) GetInt(key string) int64 { + s := strings.Trim(env.Get(key), " \t") + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return -1 + } + return val +} + +func (env *Env) SetInt(key string, value int64) { + env.Set(key, fmt.Sprintf("%d", value)) +} + +// Returns nil if key not found +func (env *Env) GetList(key string) []string { + sval := env.Get(key) + if sval == "" { + return nil + } + l := make([]string, 0, 1) + if err := json.Unmarshal([]byte(sval), &l); err != nil { + l = append(l, sval) + } + return l +} + +func (env *Env) SetJson(key string, value interface{}) error { + sval, err := json.Marshal(value) + if err != nil { + return err + } + env.Set(key, string(sval)) + return nil +} + +func (env *Env) SetList(key string, value []string) error { + return env.SetJson(key, value) +} + +func (env *Env) Set(key, value string) { + *env = append(*env, key+"="+value) +} + +func NewDecoder(src io.Reader) *Decoder { + return &Decoder{ + json.NewDecoder(src), + } +} + +type Decoder struct { + *json.Decoder +} + +func (decoder *Decoder) Decode() (*Env, error) { + m := make(map[string]interface{}) + if err := decoder.Decoder.Decode(&m); err != nil { + return nil, err + } + env := &Env{} + for key, value := range m { + env.SetAuto(key, value) + } + return env, nil +} + +// DecodeEnv decodes `src` as a json dictionary, and adds +// each decoded key-value pair to the environment. +// +// If `src` cannot be decoded as a json dictionary, an error +// is returned. +func (env *Env) Decode(src io.Reader) error { + m := make(map[string]interface{}) + if err := json.NewDecoder(src).Decode(&m); err != nil { + return err + } + for k, v := range m { + env.SetAuto(k, v) + } + return nil +} + +func (env *Env) SetAuto(k string, v interface{}) { + // FIXME: we fix-convert float values to int, because + // encoding/json decodes integers to float64, but cannot encode them back. + // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) + if fval, ok := v.(float64); ok { + env.SetInt(k, int64(fval)) + } else if sval, ok := v.(string); ok { + env.Set(k, sval) + } else if val, err := json.Marshal(v); err == nil { + env.Set(k, string(val)) + } else { + env.Set(k, fmt.Sprintf("%v", v)) + } +} + +func (env *Env) Encode(dst io.Writer) error { + m := make(map[string]interface{}) + for k, v := range env.Map() { + var val interface{} + if err := json.Unmarshal([]byte(v), &val); err == nil { + // FIXME: we fix-convert float values to int, because + // encoding/json decodes integers to float64, but cannot encode them back. + // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) + if fval, isFloat := val.(float64); isFloat { + val = int(fval) + } + m[k] = val + } else { + m[k] = v + } + } + if err := json.NewEncoder(dst).Encode(&m); err != nil { + return err + } + return nil +} + +func (env *Env) WriteTo(dst io.Writer) (n int64, err error) { + // FIXME: return the number of bytes written to respect io.WriterTo + return 0, env.Encode(dst) +} + +func (env *Env) Export(dst interface{}) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("ExportEnv %s", err) + } + }() + var buf bytes.Buffer + // step 1: encode/marshal the env to an intermediary json representation + if err := env.Encode(&buf); err != nil { + return err + } + // step 2: decode/unmarshal the intermediary json into the destination object + if err := json.NewDecoder(&buf).Decode(dst); err != nil { + return err + } + return nil +} + +func (env *Env) Import(src interface{}) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("ImportEnv: %s", err) + } + }() + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(src); err != nil { + return err + } + if err := env.Decode(&buf); err != nil { + return err + } + return nil +} + +func (env *Env) Map() map[string]string { + m := make(map[string]string) + for _, kv := range *env { + parts := strings.SplitN(kv, "=", 2) + m[parts[0]] = parts[1] + } + return m +} + diff --git a/engine/job.go b/engine/job.go index 066a144664..478f06cd60 100644 --- a/engine/job.go +++ b/engine/job.go @@ -1,11 +1,8 @@ package engine import ( - "bytes" - "encoding/json" "fmt" "io" - "strconv" "strings" "time" ) @@ -27,7 +24,7 @@ type Job struct { Eng *Engine Name string Args []string - env []string + env *Env Stdout *Output Stderr *Output Stdin *Input @@ -105,80 +102,40 @@ func (job *Job) String() string { } func (job *Job) Getenv(key string) (value string) { - for _, kv := range job.env { - if strings.Index(kv, "=") == -1 { - continue - } - parts := strings.SplitN(kv, "=", 2) - if parts[0] != key { - continue - } - if len(parts) < 2 { - value = "" - } else { - value = parts[1] - } - } - return + return job.env.Get(key) } func (job *Job) GetenvBool(key string) (value bool) { - s := strings.ToLower(strings.Trim(job.Getenv(key), " \t")) - if s == "" || s == "0" || s == "no" || s == "false" || s == "none" { - return false - } - return true + return job.env.GetBool(key) } func (job *Job) SetenvBool(key string, value bool) { - if value { - job.Setenv(key, "1") - } else { - job.Setenv(key, "0") - } + job.env.SetBool(key, value) } func (job *Job) GetenvInt(key string) int64 { - s := strings.Trim(job.Getenv(key), " \t") - val, err := strconv.ParseInt(s, 10, 64) - if err != nil { - return -1 - } - return val + return job.env.GetInt(key) } func (job *Job) SetenvInt(key string, value int64) { - job.Setenv(key, fmt.Sprintf("%d", value)) + job.env.SetInt(key, value) } // Returns nil if key not found func (job *Job) GetenvList(key string) []string { - sval := job.Getenv(key) - if sval == "" { - return nil - } - l := make([]string, 0, 1) - if err := json.Unmarshal([]byte(sval), &l); err != nil { - l = append(l, sval) - } - return l + return job.env.GetList(key) } func (job *Job) SetenvJson(key string, value interface{}) error { - sval, err := json.Marshal(value) - if err != nil { - return err - } - job.Setenv(key, string(sval)) - return nil + return job.env.SetJson(key, value) } func (job *Job) SetenvList(key string, value []string) error { - return job.SetenvJson(key, value) + return job.env.SetJson(key, value) } func (job *Job) Setenv(key, value string) { - job.env = append(job.env, key+"="+value) + job.env.Set(key, value) } // DecodeEnv decodes `src` as a json dictionary, and adds @@ -187,90 +144,23 @@ func (job *Job) Setenv(key, value string) { // If `src` cannot be decoded as a json dictionary, an error // is returned. func (job *Job) DecodeEnv(src io.Reader) error { - m := make(map[string]interface{}) - if err := json.NewDecoder(src).Decode(&m); err != nil { - return err - } - for k, v := range m { - // FIXME: we fix-convert float values to int, because - // encoding/json decodes integers to float64, but cannot encode them back. - // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) - if fval, ok := v.(float64); ok { - job.SetenvInt(k, int64(fval)) - } else if sval, ok := v.(string); ok { - job.Setenv(k, sval) - } else if val, err := json.Marshal(v); err == nil { - job.Setenv(k, string(val)) - } else { - job.Setenv(k, fmt.Sprintf("%v", v)) - } - } - return nil + return job.env.Decode(src) } func (job *Job) EncodeEnv(dst io.Writer) error { - m := make(map[string]interface{}) - for k, v := range job.Environ() { - var val interface{} - if err := json.Unmarshal([]byte(v), &val); err == nil { - // FIXME: we fix-convert float values to int, because - // encoding/json decodes integers to float64, but cannot encode them back. - // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) - if fval, isFloat := val.(float64); isFloat { - val = int(fval) - } - m[k] = val - } else { - m[k] = v - } - } - if err := json.NewEncoder(dst).Encode(&m); err != nil { - return err - } - return nil + return job.env.Encode(dst) } func (job *Job) ExportEnv(dst interface{}) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("ExportEnv %s", err) - } - }() - var buf bytes.Buffer - // step 1: encode/marshal the env to an intermediary json representation - if err := job.EncodeEnv(&buf); err != nil { - return err - } - // step 2: decode/unmarshal the intermediary json into the destination object - if err := json.NewDecoder(&buf).Decode(dst); err != nil { - return err - } - return nil + return job.env.Export(dst) } func (job *Job) ImportEnv(src interface{}) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("ImportEnv: %s", err) - } - }() - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(src); err != nil { - return err - } - if err := job.DecodeEnv(&buf); err != nil { - return err - } - return nil + return job.env.Import(src) } func (job *Job) Environ() map[string]string { - m := make(map[string]string) - for _, kv := range job.env { - parts := strings.SplitN(kv, "=", 2) - m[parts[0]] = parts[1] - } - return m + return job.env.Map() } func (job *Job) Logf(format string, args ...interface{}) (n int, err error) { From a7a171b6c2916b7be3fa51c9465516acb68c3e32 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 06:16:10 +0000 Subject: [PATCH 034/105] Engine: Output.AddEnv decodes structured data from the standard output of a job --- engine/streams.go | 27 +++++++++++++++++++++++++++ engine/streams_test.go | 20 ++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/engine/streams.go b/engine/streams.go index ff7049074a..ee26fe5f1e 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -164,3 +164,30 @@ func Tail(src io.Reader, n int, dst *[]string) { *dst = append(*dst, v.(string)) }) } + +// AddEnv starts a new goroutine which will decode all subsequent data +// as a stream of json-encoded objects, and point `dst` to the last +// decoded object. +// The result `env` can be queried using the type-neutral Env interface. +// It is not safe to query `env` until the Output is closed. +func (o *Output) AddEnv() (dst *Env, err error) { + src, err := o.AddPipe() + if err != nil { + return nil, err + } + dst = &Env{} + o.tasks.Add(1) + go func() { + defer o.tasks.Done() + decoder := NewDecoder(src) + for { + env, err := decoder.Decode() + if err != nil { + return + } + *dst= *env + } + }() + return dst, nil +} + diff --git a/engine/streams_test.go b/engine/streams_test.go index 108c9850a5..37720c61ea 100644 --- a/engine/streams_test.go +++ b/engine/streams_test.go @@ -72,6 +72,26 @@ func (w *sentinelWriteCloser) Close() error { return nil } +func TestOutputAddEnv(t *testing.T) { + input := "{\"foo\": \"bar\", \"answer_to_life_the_universe_and_everything\": 42}" + o := NewOutput() + result, err := o.AddEnv() + if err != nil { + t.Fatal(err) + } + o.Write([]byte(input)) + o.Close() + if v := result.Get("foo"); v != "bar" { + t.Errorf("Expected %v, got %v", "bar", v) + } + if v := result.GetInt("answer_to_life_the_universe_and_everything"); v != 42 { + t.Errorf("Expected %v, got %v", 42, v) + } + if v := result.Get("this-value-doesnt-exist"); v != "" { + t.Errorf("Expected %v, got %v", "", v) + } +} + func TestOutputAddClose(t *testing.T) { o := NewOutput() var s sentinelWriteCloser From f80681815422865fbe0e9b637f38553fa965004a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 07:33:23 +0000 Subject: [PATCH 035/105] Engine: convenience http transport for simple remote job execution --- engine/http.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 engine/http.go diff --git a/engine/http.go b/engine/http.go new file mode 100644 index 0000000000..6391b3ff5a --- /dev/null +++ b/engine/http.go @@ -0,0 +1,40 @@ +package engine + +import ( + "path" + "net/http" +) + +// ServeHTTP executes a job as specified by the http request `r`, and sends the +// result as an http response. +// This method allows an Engine instance to be passed as a standard http.Handler interface. +// +// Note that the protocol used in this methid is a convenience wrapper and is not the canonical +// implementation of remote job execution. This is because HTTP/1 does not handle stream multiplexing, +// and so cannot differentiate stdout from stderr. Additionally, headers cannot be added to a response +// once data has been written to the body, which makes it inconvenient to return metadata such +// as the exit status. +// +func (eng *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request) { + jobName := path.Base(r.URL.Path) + jobArgs, exists := r.URL.Query()["a"] + if !exists { + jobArgs = []string{} + } + w.Header().Set("Job-Name", jobName) + for _, arg := range(jobArgs) { + w.Header().Add("Job-Args", arg) + } + job := eng.Job(jobName, jobArgs...) + job.Stdout.Add(w) + job.Stderr.Add(w) + // FIXME: distinguish job status from engine error in Run() + // The former should be passed as a special header, the former + // should cause a 500 status + w.WriteHeader(http.StatusOK) + // The exit status cannot be sent reliably with HTTP1, because headers + // can only be sent before the body. + // (we could possibly use http footers via chunked encoding, but I couldn't find + // how to use them in net/http) + job.Run() +} From 869a11bc93c4b5dc9dbd74d65e69045f367bdb36 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 07:35:24 +0000 Subject: [PATCH 036/105] Cleanup version introspection * Unify version checking code into version.go * Make 'version' available as a job in the engine * Use simplified version checking code when setting user agent for registry client. --- server.go | 29 +++++++---------------------- version.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 version.go diff --git a/server.go b/server.go index 49b25f7098..c23e9477e3 100644 --- a/server.go +++ b/server.go @@ -144,27 +144,6 @@ func (v *simpleVersionInfo) Version() string { return v.version } -// versionCheckers() returns version informations of: -// docker, go, git-commit (of the docker) and the host's kernel. -// -// Such information will be used on call to NewRegistry(). -func (srv *Server) versionInfos() []utils.VersionInfo { - v := srv.DockerVersion() - ret := append(make([]utils.VersionInfo, 0, 4), &simpleVersionInfo{"docker", v.Version}) - - if len(v.GoVersion) > 0 { - ret = append(ret, &simpleVersionInfo{"go", v.GoVersion}) - } - if len(v.GitCommit) > 0 { - ret = append(ret, &simpleVersionInfo{"git-commit", v.GitCommit}) - } - if kernelVersion, err := utils.GetKernelVersion(); err == nil { - ret = append(ret, &simpleVersionInfo{"kernel", kernelVersion.String()}) - } - - return ret -} - // ContainerKill send signal to the container // If no signal is given (sig 0), then Kill with SIGKILL and wait // for the container to exit. @@ -1874,7 +1853,13 @@ func NewServer(eng *engine.Engine, config *DaemonConfig) (*Server, error) { func (srv *Server) HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { srv.Lock() defer srv.Unlock() - ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + v := dockerVersion() + httpVersion := make([]utils.VersionInfo, 0, 4) + httpVersion = append(httpVersion, &simpleVersionInfo{"docker", v.Get("Version")}) + httpVersion = append(httpVersion, &simpleVersionInfo{"go", v.Get("GoVersion")}) + httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", v.Get("GitCommit")}) + httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", v.Get("KernelVersion")}) + ud := utils.NewHTTPUserAgentDecorator(httpVersion...) md := &utils.HTTPMetaHeadersDecorator{ Headers: metaHeaders, } diff --git a/version.go b/version.go new file mode 100644 index 0000000000..c85e77c59a --- /dev/null +++ b/version.go @@ -0,0 +1,33 @@ +package docker + +import ( + "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/utils" + "runtime" +) + +func init() { + engine.Register("version", jobVersion) +} + +func jobVersion(job *engine.Job) engine.Status { + if _, err := dockerVersion().WriteTo(job.Stdout); err != nil { + job.Errorf("%s", err) + return engine.StatusErr + } + return engine.StatusOK +} + +// dockerVersion returns detailed version information in the form of a queriable +// environment. +func dockerVersion() *engine.Env { + v := &engine.Env{} + v.Set("Version", VERSION) + v.Set("GitCommit", GITCOMMIT) + v.Set("GoVersion", runtime.Version()) + // FIXME:utils.GetKernelVersion should only be needed here + if kernelVersion, err := utils.GetKernelVersion(); err == nil { + v.Set("KernelVersion", kernelVersion.String()) + } + return v +} From de35b346d17ef86ef091e1736575c60c41fc3771 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 8 Dec 2013 07:41:53 +0000 Subject: [PATCH 037/105] Port 'docker version' to the engine API --- api.go | 3 ++- api_params.go | 6 ------ commands.go | 25 ++++++++++++------------- integration/api_test.go | 15 +++++++++++---- server.go | 8 -------- 5 files changed, 25 insertions(+), 32 deletions(-) diff --git a/api.go b/api.go index 0aa9695702..879c688c01 100644 --- a/api.go +++ b/api.go @@ -140,7 +140,8 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque } func getVersion(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - return writeJSON(w, http.StatusOK, srv.DockerVersion()) + srv.Eng.ServeHTTP(w, r) + return nil } func postContainersKill(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/api_params.go b/api_params.go index a1bdd3e147..411cafae2e 100644 --- a/api_params.go +++ b/api_params.go @@ -95,12 +95,6 @@ type ( IP string } - APIVersion struct { - Version string - GitCommit string `json:",omitempty"` - GoVersion string `json:",omitempty"` - } - APIWait struct { StatusCode int } diff --git a/commands.go b/commands.go index 48c0241b17..1532aef769 100644 --- a/commands.go +++ b/commands.go @@ -11,6 +11,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/term" "github.com/dotcloud/docker/utils" @@ -391,26 +392,24 @@ func (cli *DockerCli) CmdVersion(args ...string) error { return err } - var out APIVersion - err = json.Unmarshal(body, &out) + out := engine.NewOutput() + remoteVersion, err := out.AddEnv() if err != nil { - utils.Errorf("Error unmarshal: body: %s, err: %s\n", body, err) + utils.Errorf("Error reading remote version: %s\n", err) return err } - if out.Version != "" { - fmt.Fprintf(cli.out, "Server version: %s\n", out.Version) + if _, err := out.Write(body); err != nil { + utils.Errorf("Error reading remote version: %s\n", err) + return err } - if out.GitCommit != "" { - fmt.Fprintf(cli.out, "Git commit (server): %s\n", out.GitCommit) - } - if out.GoVersion != "" { - fmt.Fprintf(cli.out, "Go version (server): %s\n", out.GoVersion) - } - + out.Close() + fmt.Fprintf(cli.out, "Server version: %s\n", remoteVersion.Get("Version")) + fmt.Fprintf(cli.out, "Git commit (server): %s\n", remoteVersion.Get("GitCommit")) + fmt.Fprintf(cli.out, "Go version (server): %s\n", remoteVersion.Get("GoVersion")) release := utils.GetReleaseVersion() if release != "" { fmt.Fprintf(cli.out, "Last stable version: %s", release) - if (VERSION != "" || out.Version != "") && (strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release) { + if (VERSION != "" || remoteVersion.Exists("Version")) && (strings.Trim(VERSION, "-dev") != release || strings.Trim(remoteVersion.Get("Version"), "-dev") != release) { fmt.Fprintf(cli.out, ", please update docker") } fmt.Fprintf(cli.out, "\n") diff --git a/integration/api_test.go b/integration/api_test.go index 896831c3d0..819c43a8a5 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker" + "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "io" "net" @@ -35,12 +36,18 @@ func TestGetVersion(t *testing.T) { } assertHttpNotError(r, t) - v := &docker.APIVersion{} - if err = json.Unmarshal(r.Body.Bytes(), v); err != nil { + out := engine.NewOutput() + v, err := out.AddEnv() + if err != nil { t.Fatal(err) } - if v.Version != docker.VERSION { - t.Errorf("Expected version %s, %s found", docker.VERSION, v.Version) + if _, err := io.Copy(out, r.Body); err != nil { + t.Fatal(err) + } + out.Close() + expected := docker.VERSION + if result := v.Get("Version"); result != expected { + t.Errorf("Expected version %s, %s found", expected, result) } } diff --git a/server.go b/server.go index c23e9477e3..e948de8346 100644 --- a/server.go +++ b/server.go @@ -118,14 +118,6 @@ func (srv *Server) ListenAndServe(job *engine.Job) engine.Status { return engine.StatusOK } -func (srv *Server) DockerVersion() APIVersion { - return APIVersion{ - Version: VERSION, - GitCommit: GITCOMMIT, - GoVersion: runtime.Version(), - } -} - // simpleVersionInfo is a simple implementation of // the interface VersionInfo, which is used // to provide version information for some product, From f832b76bdfafe3108af61d00171567da02482222 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Sat, 7 Dec 2013 16:53:39 +0530 Subject: [PATCH 038/105] archlinux installation doc: correct some details 1. The AUR package is called docker-git, not lxc-docker-git. 2. According to the official community package, docker depends on sqlite. 3. 02ef8ec (Update archlinux.rst as packages have changed, 2013-12-06) updated the installation instructions, but left behind residual wording about the AUR package not being officially supported; the community repository is officially supported. Signed-off-by: Ramkumar Ramachandra --- docs/sources/installation/archlinux.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/installation/archlinux.rst b/docs/sources/installation/archlinux.rst index 44249b178f..3859317c44 100644 --- a/docs/sources/installation/archlinux.rst +++ b/docs/sources/installation/archlinux.rst @@ -11,8 +11,7 @@ Arch Linux .. include:: install_unofficial.inc -Installing on Arch Linux is not officially supported but can be handled via -the package in community: +Installing on Arch Linux can be handled via the package in community: * `docker `_ @@ -33,6 +32,7 @@ the packages. The core dependencies are: * device-mapper * iproute2 * lxc +* sqlite Installation @@ -48,7 +48,7 @@ is all that is needed. For the AUR package execute: :: - yaourt -S lxc-docker-git + yaourt -S docker-git The instructions here assume **yaourt** is installed. See `Arch User Repository `_ From 416b16e1e200420644072b354b9595ffa9474682 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 12:57:11 -0700 Subject: [PATCH 039/105] Simplify and resync hack/make/test and hack/make/dyntest output handling --- hack/make/dyntest | 18 ++++++++++++++++-- hack/make/test | 11 +++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/hack/make/dyntest b/hack/make/dyntest index 61f03ada1d..ae157dff55 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -10,6 +10,10 @@ if [ ! -x "$INIT" ]; then false fi +TEXTRESET=$'\033[0m' # reset the foreground colour +RED=$'\033[31m' +GREEN=$'\033[32m' + # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. @@ -37,16 +41,26 @@ bundle_test() { go test -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS ); then TESTS_FAILED+=("$test_dir") + echo + echo "${RED}Tests failed: $test_dir${TEXTRESET}" sleep 1 # give it a second, so observers watching can take note fi done - + + echo + echo + echo + # if some tests fail, we want the bundlescript to fail, but we want to # try running ALL the tests first, hence TESTS_FAILED if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then + echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" echo - echo "Test failures in: ${TESTS_FAILED[@]}" false + else + echo "${GREEN}Test success${TEXTRESET}" + echo + true fi } 2>&1 | tee $DEST/test.log } diff --git a/hack/make/test b/hack/make/test index c8fae70535..ad93e91cf8 100644 --- a/hack/make/test +++ b/hack/make/test @@ -35,21 +35,24 @@ bundle_test() { ); then TESTS_FAILED+=("$test_dir") echo - echo "${RED}Test Failed: $test_dir${TEXTRESET}" - echo + echo "${RED}Tests failed: $test_dir${TEXTRESET}" sleep 1 # give it a second, so observers watching can take note fi done + echo + echo + echo + # if some tests fail, we want the bundlescript to fail, but we want to # try running ALL the tests first, hence TESTS_FAILED if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then - echo echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" + echo false else - echo echo "${GREEN}Test success${TEXTRESET}" + echo true fi } 2>&1 | tee $DEST/test.log From dcfc4ada4dcc8b1f878370976f29ca0485e690f8 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 13:49:57 -0700 Subject: [PATCH 040/105] Clean output and simplify hack/make/*test by adding go_test_dir function in make.sh --- hack/make.sh | 18 ++++++++++++++++++ hack/make/dyntest | 23 +++++++---------------- hack/make/test | 14 ++------------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index 6139c93bb5..23b77509b6 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -62,6 +62,24 @@ LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w' LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' BUILDFLAGS='-tags netgo' +# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. +# You can use this to select certain tests to run, eg. +# +# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test +# +go_test_dir() { + dir=$1 + ( # we run "go test -i" ouside the "set -x" to provde cleaner output + cd "$dir" + go test -i -ldflags "$LDFLAGS" $BUILDFLAGS + ) + ( + set -x + cd "$dir" + go test -ldflags "$LDFLAGS" $BUILDFLAGS $TESTFLAGS + ) +} + bundle() { bundlescript=$1 bundle=$(basename $bundlescript) diff --git a/hack/make/dyntest b/hack/make/dyntest index ae157dff55..e6e1dbb5b2 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -17,29 +17,20 @@ GREEN=$'\033[32m' # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. -# -# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test +# +# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test # bundle_test() { { date - + + export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION + TESTS_FAILED=() for test_dir in $(find_test_dirs); do echo - - if ! ( - set -x - cd $test_dir - - # Install packages that are dependencies of the tests. - # Note: Does not run the tests. - go test -i -ldflags "$LDFLAGS" $BUILDFLAGS - - # Run the tests with the optional $TESTFLAGS. - export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION - go test -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS - ); then + + if ! LDFLAGS="$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" go_test_dir "$test_dir"; then TESTS_FAILED+=("$test_dir") echo echo "${RED}Tests failed: $test_dir${TEXTRESET}" diff --git a/hack/make/test b/hack/make/test index ad93e91cf8..9ec86f0780 100644 --- a/hack/make/test +++ b/hack/make/test @@ -12,7 +12,7 @@ GREEN=$'\033[32m' # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. # -# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test +# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test # bundle_test() { { @@ -22,17 +22,7 @@ bundle_test() { for test_dir in $(find_test_dirs); do echo - if ! ( - set -x - cd $test_dir - - # Install packages that are dependencies of the tests. - # Note: Does not run the tests. - go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS - - # Run the tests with the optional $TESTFLAGS. - go test -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS - ); then + if ! LDFLAGS="$LDFLAGS $LDFLAGS_STATIC" go_test_dir "$test_dir"; then TESTS_FAILED+=("$test_dir") echo echo "${RED}Tests failed: $test_dir${TEXTRESET}" From bac3a8e6f5eca31108787b98d5659523eefa6a30 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 13:50:48 -0700 Subject: [PATCH 041/105] Add much better pruning of non-tested directories, including pruning the integration tests directory (doing more with "find" and nothing with "grep") --- hack/make/dyntest | 7 ++++--- hack/make/test | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hack/make/dyntest b/hack/make/dyntest index e6e1dbb5b2..f7290c9557 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -61,9 +61,10 @@ bundle_test() { # holding Go test files, and prints their paths on standard output, one per # line. find_test_dirs() { - find . -name '*_test.go' | grep -v '^./vendor' | - { while read f; do dirname $f; done; } | - sort -u + find -not \( \ + \( -wholename './vendor' -o -wholename './integration' \) \ + -prune \ + \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u } bundle_test diff --git a/hack/make/test b/hack/make/test index 9ec86f0780..760c5a5fc6 100644 --- a/hack/make/test +++ b/hack/make/test @@ -53,9 +53,10 @@ bundle_test() { # holding Go test files, and prints their paths on standard output, one per # line. find_test_dirs() { - find . -name '*_test.go' | grep -v '^./vendor' | - { while read f; do dirname $f; done; } | - sort -u + find -not \( \ + \( -wholename './vendor' -o -wholename './integration' \) \ + -prune \ + \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u } bundle_test From c094807a1b8159bf237091ecc85fcbd6e2092a3e Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 6 Dec 2013 23:35:26 -0500 Subject: [PATCH 042/105] Added section on Trusted Builds --- docs/sources/use/workingwithrepository.rst | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/sources/use/workingwithrepository.rst b/docs/sources/use/workingwithrepository.rst index 5faebcc2e5..3797f6ab52 100644 --- a/docs/sources/use/workingwithrepository.rst +++ b/docs/sources/use/workingwithrepository.rst @@ -152,6 +152,41 @@ or tag. .. _using_private_repositories: +Trusted Builds +-------------- + +Trusted Builds automate the building and updating of images from GitHub, directly +on docker.io servers. It works by adding a commit hook to your selected repository, +triggering a build and update when you push a commit. + +To setup a trusted build +++++++++++++++++++++++++ + +#. Create a `Docker Index account `_ and login. +#. Link your GitHub account through the ``Link Accounts`` menu. +#. `Configure a Trusted build `_. +#. Pick a GitHub project that has a ``Dockerfile`` that you want to build. +#. Pick the branch you want to build (the default is the ``master`` branch). +#. Give the Trusted Build a name. +#. Assign an optional Docker tag to the Build. +#. Specify where the ``Dockerfile`` is located. The default is ``/``. + +Once the Trusted Build is configured it will automatically trigger a build, and +in a few minutes, if there are no errors, you will see your new trusted build +on the Docker Index. It will will stay in sync with your GitHub repo until you +deactivate the Trusted Build. + +If you want to see the status of your Trusted Builds you can go to your +`Trusted Builds page `_ on the Docker index, +and it will show you the status of your builds, and the build history. + +Once you've created a Trusted Build you can deactive or delete it. You cannot +however push to a Trusted Build with the ``docker push`` command. You can only +manage it by committing code to your GitHub repository. + +You can create multiple Trusted Builds per repository and configure them to +point to specific ``Dockerfile``'s or Git branches. + Private Repositories -------------------- From 8ec96c9605b1e97fb36f3c2a2d85632f62fb047b Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 9 Dec 2013 00:02:13 +0200 Subject: [PATCH 043/105] remove vendored dotcloud/tar The tar dependency has been removed. It's time to remove the vendored tar as well. --- vendor/src/github.com/dotcloud/tar/LICENSE | 27 -- vendor/src/github.com/dotcloud/tar/README | 3 - vendor/src/github.com/dotcloud/tar/common.go | 299 ------------- .../github.com/dotcloud/tar/example_test.go | 79 ---- vendor/src/github.com/dotcloud/tar/reader.go | 396 ------------------ .../github.com/dotcloud/tar/reader_test.go | 385 ----------------- .../src/github.com/dotcloud/tar/stat_atim.go | 20 - .../github.com/dotcloud/tar/stat_atimespec.go | 20 - .../src/github.com/dotcloud/tar/stat_unix.go | 32 -- .../src/github.com/dotcloud/tar/tar_test.go | 271 ------------ .../github.com/dotcloud/tar/testdata/gnu.tar | Bin 3072 -> 0 bytes .../dotcloud/tar/testdata/nil-uid.tar | Bin 1024 -> 0 bytes .../github.com/dotcloud/tar/testdata/pax.tar | Bin 10240 -> 0 bytes .../dotcloud/tar/testdata/small.txt | 1 - .../dotcloud/tar/testdata/small2.txt | 1 - .../github.com/dotcloud/tar/testdata/star.tar | Bin 3072 -> 0 bytes .../dotcloud/tar/testdata/ustar.tar | Bin 2048 -> 0 bytes .../github.com/dotcloud/tar/testdata/v7.tar | Bin 3584 -> 0 bytes .../dotcloud/tar/testdata/writer-big.tar | Bin 4096 -> 0 bytes .../dotcloud/tar/testdata/writer.tar | Bin 3584 -> 0 bytes vendor/src/github.com/dotcloud/tar/writer.go | 377 ----------------- .../github.com/dotcloud/tar/writer_test.go | 393 ----------------- 22 files changed, 2304 deletions(-) delete mode 100644 vendor/src/github.com/dotcloud/tar/LICENSE delete mode 100644 vendor/src/github.com/dotcloud/tar/README delete mode 100644 vendor/src/github.com/dotcloud/tar/common.go delete mode 100644 vendor/src/github.com/dotcloud/tar/example_test.go delete mode 100644 vendor/src/github.com/dotcloud/tar/reader.go delete mode 100644 vendor/src/github.com/dotcloud/tar/reader_test.go delete mode 100644 vendor/src/github.com/dotcloud/tar/stat_atim.go delete mode 100644 vendor/src/github.com/dotcloud/tar/stat_atimespec.go delete mode 100644 vendor/src/github.com/dotcloud/tar/stat_unix.go delete mode 100644 vendor/src/github.com/dotcloud/tar/tar_test.go delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/gnu.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/nil-uid.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/pax.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/small.txt delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/small2.txt delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/star.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/ustar.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/v7.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/writer-big.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/testdata/writer.tar delete mode 100644 vendor/src/github.com/dotcloud/tar/writer.go delete mode 100644 vendor/src/github.com/dotcloud/tar/writer_test.go diff --git a/vendor/src/github.com/dotcloud/tar/LICENSE b/vendor/src/github.com/dotcloud/tar/LICENSE deleted file mode 100644 index 7448756763..0000000000 --- a/vendor/src/github.com/dotcloud/tar/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/src/github.com/dotcloud/tar/README b/vendor/src/github.com/dotcloud/tar/README deleted file mode 100644 index 04a293bbca..0000000000 --- a/vendor/src/github.com/dotcloud/tar/README +++ /dev/null @@ -1,3 +0,0 @@ -This is a fork of the upstream Go [archive/tar](http://golang.org/pkg/archive/tar/) package to add PAX header support. - -You can monitor the upstream pull request [here](https://codereview.appspot.com/12561043/). diff --git a/vendor/src/github.com/dotcloud/tar/common.go b/vendor/src/github.com/dotcloud/tar/common.go deleted file mode 100644 index 693076efce..0000000000 --- a/vendor/src/github.com/dotcloud/tar/common.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package tar implements access to tar archives. -// It aims to cover most of the variations, including those produced -// by GNU and BSD tars. -// -// References: -// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 -// http://www.gnu.org/software/tar/manual/html_node/Standard.html -// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html -package tar - -import ( - "bytes" - "errors" - "fmt" - "os" - "path" - "time" -) - -const ( - blockSize = 512 - - // Types - TypeReg = '0' // regular file - TypeRegA = '\x00' // regular file - TypeLink = '1' // hard link - TypeSymlink = '2' // symbolic link - TypeChar = '3' // character device node - TypeBlock = '4' // block device node - TypeDir = '5' // directory - TypeFifo = '6' // fifo node - TypeCont = '7' // reserved - TypeXHeader = 'x' // extended header - TypeXGlobalHeader = 'g' // global extended header - TypeGNULongName = 'L' // Next file has a long name - TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name -) - -// A Header represents a single header in a tar archive. -// Some fields may not be populated. -type Header struct { - Name string // name of header file entry - Mode int64 // permission and mode bits - Uid int // user id of owner - Gid int // group id of owner - Size int64 // length in bytes - ModTime time.Time // modified time - Typeflag byte // type of header entry - Linkname string // target name of link - Uname string // user name of owner - Gname string // group name of owner - Devmajor int64 // major number of character or block device - Devminor int64 // minor number of character or block device - AccessTime time.Time // access time - ChangeTime time.Time // status change time -} - -// File name constants from the tar spec. -const ( - fileNameSize = 100 // Maximum number of bytes in a standard tar name. - fileNamePrefixSize = 155 // Maximum number of ustar extension bytes. -) - -// FileInfo returns an os.FileInfo for the Header. -func (h *Header) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - h *Header -} - -func (fi headerFileInfo) Size() int64 { return fi.h.Size } -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } -func (fi headerFileInfo) Sys() interface{} { return fi.h } - -// Name returns the base name of the file. -func (fi headerFileInfo) Name() string { - if fi.IsDir() { - return path.Clean(fi.h.Name) - } - return fi.h.Name -} - -// Mode returns the permission and mode bits for the headerFileInfo. -func (fi headerFileInfo) Mode() (mode os.FileMode) { - // Set file permission bits. - mode = os.FileMode(fi.h.Mode).Perm() - - // Set setuid, setgid and sticky bits. - if fi.h.Mode&c_ISUID != 0 { - // setuid - mode |= os.ModeSetuid - } - if fi.h.Mode&c_ISGID != 0 { - // setgid - mode |= os.ModeSetgid - } - if fi.h.Mode&c_ISVTX != 0 { - // sticky - mode |= os.ModeSticky - } - - // Set file mode bits. - // clear perm, setuid, setgid and sticky bits. - m := os.FileMode(fi.h.Mode) &^ 07777 - if m == c_ISDIR { - // directory - mode |= os.ModeDir - } - if m == c_ISFIFO { - // named pipe (FIFO) - mode |= os.ModeNamedPipe - } - if m == c_ISLNK { - // symbolic link - mode |= os.ModeSymlink - } - if m == c_ISBLK { - // device file - mode |= os.ModeDevice - } - if m == c_ISCHR { - // Unix character device - mode |= os.ModeDevice - mode |= os.ModeCharDevice - } - if m == c_ISSOCK { - // Unix domain socket - mode |= os.ModeSocket - } - - switch fi.h.Typeflag { - case TypeLink, TypeSymlink: - // hard link, symbolic link - mode |= os.ModeSymlink - case TypeChar: - // character device node - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case TypeBlock: - // block device node - mode |= os.ModeDevice - case TypeDir: - // directory - mode |= os.ModeDir - case TypeFifo: - // fifo node - mode |= os.ModeNamedPipe - } - - return mode -} - -// sysStat, if non-nil, populates h from system-dependent fields of fi. -var sysStat func(fi os.FileInfo, h *Header) error - -// Mode constants from the tar spec. -const ( - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) - c_ISDIR = 040000 // Directory - c_ISFIFO = 010000 // FIFO - c_ISREG = 0100000 // Regular file - c_ISLNK = 0120000 // Symbolic link - c_ISBLK = 060000 // Block special file - c_ISCHR = 020000 // Character special file - c_ISSOCK = 0140000 // Socket -) - -// Keywords for the PAX Extended Header -const ( - paxAtime = "atime" - paxCharset = "charset" - paxComment = "comment" - paxCtime = "ctime" // please note that ctime is not a valid pax header. - paxGid = "gid" - paxGname = "gname" - paxLinkpath = "linkpath" - paxMtime = "mtime" - paxPath = "path" - paxSize = "size" - paxUid = "uid" - paxUname = "uname" - paxNone = "" -) - -// FileInfoHeader creates a partially-populated Header from fi. -// If fi describes a symlink, FileInfoHeader records link as the link target. -// If fi describes a directory, a slash is appended to the name. -func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { - if fi == nil { - return nil, errors.New("tar: FileInfo is nil") - } - fm := fi.Mode() - h := &Header{ - Name: fi.Name(), - ModTime: fi.ModTime(), - Mode: int64(fm.Perm()), // or'd with c_IS* constants later - } - switch { - case fm.IsRegular(): - h.Mode |= c_ISREG - h.Typeflag = TypeReg - h.Size = fi.Size() - case fi.IsDir(): - h.Typeflag = TypeDir - h.Mode |= c_ISDIR - h.Name += "/" - case fm&os.ModeSymlink != 0: - h.Typeflag = TypeSymlink - h.Mode |= c_ISLNK - h.Linkname = link - case fm&os.ModeDevice != 0: - if fm&os.ModeCharDevice != 0 { - h.Mode |= c_ISCHR - h.Typeflag = TypeChar - } else { - h.Mode |= c_ISBLK - h.Typeflag = TypeBlock - } - case fm&os.ModeNamedPipe != 0: - h.Typeflag = TypeFifo - h.Mode |= c_ISFIFO - case fm&os.ModeSocket != 0: - h.Mode |= c_ISSOCK - default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) - } - if fm&os.ModeSetuid != 0 { - h.Mode |= c_ISUID - } - if fm&os.ModeSetgid != 0 { - h.Mode |= c_ISGID - } - if fm&os.ModeSticky != 0 { - h.Mode |= c_ISVTX - } - if sysStat != nil { - return h, sysStat(fi, h) - } - return h, nil -} - -var zeroBlock = make([]byte, blockSize) - -// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. -// We compute and return both. -func checksum(header []byte) (unsigned int64, signed int64) { - for i := 0; i < len(header); i++ { - if i == 148 { - // The chksum field (header[148:156]) is special: it should be treated as space bytes. - unsigned += ' ' * 8 - signed += ' ' * 8 - i += 7 - continue - } - unsigned += int64(header[i]) - signed += int64(int8(header[i])) - } - return -} - -type slicer []byte - -func (sp *slicer) next(n int) (b []byte) { - s := *sp - b, *sp = s[0:n], s[n:] - return -} - -func isASCII(s string) bool { - for _, c := range s { - if c >= 0x80 { - return false - } - } - return true -} - -func toASCII(s string) string { - if isASCII(s) { - return s - } - var buf bytes.Buffer - for _, c := range s { - if c < 0x80 { - buf.WriteByte(byte(c)) - } - } - return buf.String() -} diff --git a/vendor/src/github.com/dotcloud/tar/example_test.go b/vendor/src/github.com/dotcloud/tar/example_test.go deleted file mode 100644 index 351eaa0e6c..0000000000 --- a/vendor/src/github.com/dotcloud/tar/example_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar_test - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "log" - "os" -) - -func Example() { - // Create a buffer to write our archive to. - buf := new(bytes.Buffer) - - // Create a new tar archive. - tw := tar.NewWriter(buf) - - // Add some files to the archive. - var files = []struct { - Name, Body string - }{ - {"readme.txt", "This archive contains some text files."}, - {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, - {"todo.txt", "Get animal handling licence."}, - } - for _, file := range files { - hdr := &tar.Header{ - Name: file.Name, - Size: int64(len(file.Body)), - } - if err := tw.WriteHeader(hdr); err != nil { - log.Fatalln(err) - } - if _, err := tw.Write([]byte(file.Body)); err != nil { - log.Fatalln(err) - } - } - // Make sure to check the error on Close. - if err := tw.Close(); err != nil { - log.Fatalln(err) - } - - // Open the tar archive for reading. - r := bytes.NewReader(buf.Bytes()) - tr := tar.NewReader(r) - - // Iterate through the files in the archive. - for { - hdr, err := tr.Next() - if err == io.EOF { - // end of tar archive - break - } - if err != nil { - log.Fatalln(err) - } - fmt.Printf("Contents of %s:\n", hdr.Name) - if _, err := io.Copy(os.Stdout, tr); err != nil { - log.Fatalln(err) - } - fmt.Println() - } - - // Output: - // Contents of readme.txt: - // This archive contains some text files. - // Contents of gopher.txt: - // Gopher names: - // George - // Geoffrey - // Gonzo - // Contents of todo.txt: - // Get animal handling licence. -} diff --git a/vendor/src/github.com/dotcloud/tar/reader.go b/vendor/src/github.com/dotcloud/tar/reader.go deleted file mode 100644 index b2d62f3c51..0000000000 --- a/vendor/src/github.com/dotcloud/tar/reader.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -// TODO(dsymonds): -// - pax extensions - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "os" - "strconv" - "strings" - "time" -) - -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") -) - -const maxNanoSecondIntSize = 9 - -// A Reader provides sequential access to the contents of a tar archive. -// A tar archive consists of a sequence of files. -// The Next method advances to the next file in the archive (including the first), -// and then it can be treated as an io.Reader to access the file's data. -type Reader struct { - r io.Reader - err error - nb int64 // number of unread bytes for current file entry - pad int64 // amount of padding (ignored) after current file entry -} - -// NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { return &Reader{r: r} } - -// Next advances to the next entry in the tar archive. -func (tr *Reader) Next() (*Header, error) { - var hdr *Header - if tr.err == nil { - tr.skipUnread() - } - if tr.err != nil { - return hdr, tr.err - } - hdr = tr.readHeader() - if hdr == nil { - return hdr, tr.err - } - // Check for PAX/GNU header. - switch hdr.Typeflag { - case TypeXHeader: - // PAX extended header - headers, err := parsePAX(tr) - if err != nil { - return nil, err - } - // We actually read the whole file, - // but this skips alignment padding - tr.skipUnread() - hdr = tr.readHeader() - mergePAX(hdr, headers) - return hdr, nil - case TypeGNULongName: - // We have a GNU long name header. Its contents are the real file name. - realname, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - hdr, err := tr.Next() - hdr.Name = cString(realname) - return hdr, err - case TypeGNULongLink: - // We have a GNU long link header. - realname, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - hdr, err := tr.Next() - hdr.Linkname = cString(realname) - return hdr, err - } - return hdr, tr.err -} - -// mergePAX merges well known headers according to PAX standard. -// In general headers with the same name as those found -// in the header struct overwrite those found in the header -// struct with higher precision or longer values. Esp. useful -// for name and linkname fields. -func mergePAX(hdr *Header, headers map[string]string) error { - for k, v := range headers { - switch k { - case paxPath: - hdr.Name = v - case paxLinkpath: - hdr.Linkname = v - case paxGname: - hdr.Gname = v - case paxUname: - hdr.Uname = v - case paxUid: - uid, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Uid = int(uid) - case paxGid: - gid, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Gid = int(gid) - case paxAtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.AccessTime = t - case paxMtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.ModTime = t - case paxCtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.ChangeTime = t - case paxSize: - size, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Size = int64(size) - } - - } - return nil -} - -// parsePAXTime takes a string of the form %d.%d as described in -// the PAX specification. -func parsePAXTime(t string) (time.Time, error) { - buf := []byte(t) - pos := bytes.IndexByte(buf, '.') - var seconds, nanoseconds int64 - var err error - if pos == -1 { - seconds, err = strconv.ParseInt(t, 10, 0) - if err != nil { - return time.Time{}, err - } - } else { - seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0) - if err != nil { - return time.Time{}, err - } - nano_buf := string(buf[pos+1:]) - // Pad as needed before converting to a decimal. - // For example .030 -> .030000000 -> 30000000 nanoseconds - if len(nano_buf) < maxNanoSecondIntSize { - // Right pad - nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf)) - } else if len(nano_buf) > maxNanoSecondIntSize { - // Right truncate - nano_buf = nano_buf[:maxNanoSecondIntSize] - } - nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0) - if err != nil { - return time.Time{}, err - } - } - ts := time.Unix(seconds, nanoseconds) - return ts, nil -} - -// parsePAX parses PAX headers. -// If an extended header (type 'x') is invalid, ErrHeader is returned -func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - headers := make(map[string]string) - // Each record is constructed as - // "%d %s=%s\n", length, keyword, value - for len(buf) > 0 { - // or the header was empty to start with. - var sp int - // The size field ends at the first space. - sp = bytes.IndexByte(buf, ' ') - if sp == -1 { - return nil, ErrHeader - } - // Parse the first token as a decimal integer. - n, err := strconv.ParseInt(string(buf[:sp]), 10, 0) - if err != nil { - return nil, ErrHeader - } - // Extract everything between the decimal and the n -1 on the - // beginning to to eat the ' ', -1 on the end to skip the newline. - var record []byte - record, buf = buf[sp+1:n-1], buf[n:] - // The first equals is guaranteed to mark the end of the key. - // Everything else is value. - eq := bytes.IndexByte(record, '=') - if eq == -1 { - return nil, ErrHeader - } - key, value := record[:eq], record[eq+1:] - headers[string(key)] = string(value) - } - return headers, nil -} - -// cString parses bytes as a NUL-terminated C-style string. -// If a NUL byte is not found then the whole slice is returned as a string. -func cString(b []byte) string { - n := 0 - for n < len(b) && b[n] != 0 { - n++ - } - return string(b[0:n]) -} - -func (tr *Reader) octal(b []byte) int64 { - // Check for binary format first. - if len(b) > 0 && b[0]&0x80 != 0 { - var x int64 - for i, c := range b { - if i == 0 { - c &= 0x7f // ignore signal bit in first byte - } - x = x<<8 | int64(c) - } - return x - } - - // Because unused fields are filled with NULs, we need - // to skip leading NULs. Fields may also be padded with - // spaces or NULs. - // So we remove leading and trailing NULs and spaces to - // be sure. - b = bytes.Trim(b, " \x00") - - if len(b) == 0 { - return 0 - } - x, err := strconv.ParseUint(cString(b), 8, 64) - if err != nil { - tr.err = err - } - return int64(x) -} - -// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding. -func (tr *Reader) skipUnread() { - nr := tr.nb + tr.pad // number of bytes to skip - tr.nb, tr.pad = 0, 0 - if sr, ok := tr.r.(io.Seeker); ok { - if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil { - return - } - } - _, tr.err = io.CopyN(ioutil.Discard, tr.r, nr) -} - -func (tr *Reader) verifyChecksum(header []byte) bool { - if tr.err != nil { - return false - } - - given := tr.octal(header[148:156]) - unsigned, signed := checksum(header) - return given == unsigned || given == signed -} - -func (tr *Reader) readHeader() *Header { - header := make([]byte, blockSize) - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { - return nil - } - - // Two blocks of zero bytes marks the end of the archive. - if bytes.Equal(header, zeroBlock[0:blockSize]) { - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { - return nil - } - if bytes.Equal(header, zeroBlock[0:blockSize]) { - tr.err = io.EOF - } else { - tr.err = ErrHeader // zero block and then non-zero block - } - return nil - } - - if !tr.verifyChecksum(header) { - tr.err = ErrHeader - return nil - } - - // Unpack - hdr := new(Header) - s := slicer(header) - - hdr.Name = cString(s.next(100)) - hdr.Mode = tr.octal(s.next(8)) - hdr.Uid = int(tr.octal(s.next(8))) - hdr.Gid = int(tr.octal(s.next(8))) - hdr.Size = tr.octal(s.next(12)) - hdr.ModTime = time.Unix(tr.octal(s.next(12)), 0) - s.next(8) // chksum - hdr.Typeflag = s.next(1)[0] - hdr.Linkname = cString(s.next(100)) - - // The remainder of the header depends on the value of magic. - // The original (v7) version of tar had no explicit magic field, - // so its magic bytes, like the rest of the block, are NULs. - magic := string(s.next(8)) // contains version field as well. - var format string - switch magic { - case "ustar\x0000": // POSIX tar (1003.1-1988) - if string(header[508:512]) == "tar\x00" { - format = "star" - } else { - format = "posix" - } - case "ustar \x00": // old GNU tar - format = "gnu" - } - - switch format { - case "posix", "gnu", "star": - hdr.Uname = cString(s.next(32)) - hdr.Gname = cString(s.next(32)) - devmajor := s.next(8) - devminor := s.next(8) - if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { - hdr.Devmajor = tr.octal(devmajor) - hdr.Devminor = tr.octal(devminor) - } - var prefix string - switch format { - case "posix", "gnu": - prefix = cString(s.next(155)) - case "star": - prefix = cString(s.next(131)) - hdr.AccessTime = time.Unix(tr.octal(s.next(12)), 0) - hdr.ChangeTime = time.Unix(tr.octal(s.next(12)), 0) - } - if len(prefix) > 0 { - hdr.Name = prefix + "/" + hdr.Name - } - } - - if tr.err != nil { - tr.err = ErrHeader - return nil - } - - // Maximum value of hdr.Size is 64 GB (12 octal digits), - // so there's no risk of int64 overflowing. - tr.nb = int64(hdr.Size) - tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two - - return hdr -} - -// Read reads from the current entry in the tar archive. -// It returns 0, io.EOF when it reaches the end of that entry, -// until Next is called to advance to the next entry. -func (tr *Reader) Read(b []byte) (n int, err error) { - if tr.nb == 0 { - // file consumed - return 0, io.EOF - } - - if int64(len(b)) > tr.nb { - b = b[0:tr.nb] - } - n, err = tr.r.Read(b) - tr.nb -= int64(n) - - if err == io.EOF && tr.nb > 0 { - err = io.ErrUnexpectedEOF - } - tr.err = err - return -} diff --git a/vendor/src/github.com/dotcloud/tar/reader_test.go b/vendor/src/github.com/dotcloud/tar/reader_test.go deleted file mode 100644 index 1285616565..0000000000 --- a/vendor/src/github.com/dotcloud/tar/reader_test.go +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "crypto/md5" - "fmt" - "io" - "os" - "reflect" - "strings" - "testing" - "time" -) - -type untarTest struct { - file string - headers []*Header - cksums []string -} - -var gnuTarTest = &untarTest{ - file: "testdata/gnu.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244428340, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - { - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244436044, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - }, - cksums: []string{ - "e38b27eaccb4391bdec553a7f3ae6b2f", - "c65bd2e50a56a2138bf1716f2fd56fe9", - }, -} - -var untarTests = []*untarTest{ - gnuTarTest, - { - file: "testdata/star.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244592783, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - AccessTime: time.Unix(1244592783, 0), - ChangeTime: time.Unix(1244592783, 0), - }, - { - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244592783, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - AccessTime: time.Unix(1244592783, 0), - ChangeTime: time.Unix(1244592783, 0), - }, - }, - }, - { - file: "testdata/v7.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0444, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244593104, 0), - Typeflag: '\x00', - }, - { - Name: "small2.txt", - Mode: 0444, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244593104, 0), - Typeflag: '\x00', - }, - }, - }, - { - file: "testdata/pax.tar", - headers: []*Header{ - { - Name: "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", - Mode: 0664, - Uid: 1000, - Gid: 1000, - Uname: "shane", - Gname: "shane", - Size: 7, - ModTime: time.Unix(1350244992, 23960108), - ChangeTime: time.Unix(1350244992, 23960108), - AccessTime: time.Unix(1350244992, 23960108), - Typeflag: TypeReg, - }, - { - Name: "a/b", - Mode: 0777, - Uid: 1000, - Gid: 1000, - Uname: "shane", - Gname: "shane", - Size: 0, - ModTime: time.Unix(1350266320, 910238425), - ChangeTime: time.Unix(1350266320, 910238425), - AccessTime: time.Unix(1350266320, 910238425), - Typeflag: TypeSymlink, - Linkname: "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", - }, - }, - }, - { - file: "testdata/nil-uid.tar", // golang.org/issue/5290 - headers: []*Header{ - { - Name: "P1050238.JPG.log", - Mode: 0664, - Uid: 0, - Gid: 0, - Size: 14, - ModTime: time.Unix(1365454838, 0), - Typeflag: TypeReg, - Linkname: "", - Uname: "eyefi", - Gname: "eyefi", - Devmajor: 0, - Devminor: 0, - }, - }, - }, -} - -func TestReader(t *testing.T) { -testLoop: - for i, test := range untarTests { - f, err := os.Open(test.file) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - continue - } - defer f.Close() - tr := NewReader(f) - for j, header := range test.headers { - hdr, err := tr.Next() - if err != nil || hdr == nil { - t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) - f.Close() - continue testLoop - } - if *hdr != *header { - t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", - i, j, *hdr, *header) - } - } - hdr, err := tr.Next() - if err == io.EOF { - continue testLoop - } - if hdr != nil || err != nil { - t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err) - } - } -} - -func TestPartialRead(t *testing.T) { - f, err := os.Open("testdata/gnu.tar") - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - - // Read the first four bytes; Next() should skip the last byte. - hdr, err := tr.Next() - if err != nil || hdr == nil { - t.Fatalf("Didn't get first file: %v", err) - } - buf := make([]byte, 4) - if _, err := io.ReadFull(tr, buf); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if expected := []byte("Kilt"); !bytes.Equal(buf, expected) { - t.Errorf("Contents = %v, want %v", buf, expected) - } - - // Second file - hdr, err = tr.Next() - if err != nil || hdr == nil { - t.Fatalf("Didn't get second file: %v", err) - } - buf = make([]byte, 6) - if _, err := io.ReadFull(tr, buf); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if expected := []byte("Google"); !bytes.Equal(buf, expected) { - t.Errorf("Contents = %v, want %v", buf, expected) - } -} - -func TestIncrementalRead(t *testing.T) { - test := gnuTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - - headers := test.headers - cksums := test.cksums - nread := 0 - - // loop over all files - for ; ; nread++ { - hdr, err := tr.Next() - if hdr == nil || err == io.EOF { - break - } - - // check the header - if *hdr != *headers[nread] { - t.Errorf("Incorrect header:\nhave %+v\nwant %+v", - *hdr, headers[nread]) - } - - // read file contents in little chunks EOF, - // checksumming all the way - h := md5.New() - rdbuf := make([]uint8, 8) - for { - nr, err := tr.Read(rdbuf) - if err == io.EOF { - break - } - if err != nil { - t.Errorf("Read: unexpected error %v\n", err) - break - } - h.Write(rdbuf[0:nr]) - } - // verify checksum - have := fmt.Sprintf("%x", h.Sum(nil)) - want := cksums[nread] - if want != have { - t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) - } - } - if nread != len(headers) { - t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) - } -} - -func TestNonSeekable(t *testing.T) { - test := gnuTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - type readerOnly struct { - io.Reader - } - tr := NewReader(readerOnly{f}) - nread := 0 - - for ; ; nread++ { - _, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - } - - if nread != len(test.headers) { - t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread) - } -} - -func TestParsePAXHeader(t *testing.T) { - paxTests := [][3]string{ - {"a", "a=name", "10 a=name\n"}, // Test case involving multiple acceptable lengths - {"a", "a=name", "9 a=name\n"}, // Test case involving multiple acceptable length - {"mtime", "mtime=1350244992.023960108", "30 mtime=1350244992.023960108\n"}} - for _, test := range paxTests { - key, expected, raw := test[0], test[1], test[2] - reader := bytes.NewBuffer([]byte(raw)) - headers, err := parsePAX(reader) - if err != nil { - t.Errorf("Couldn't parse correctly formatted headers: %v", err) - continue - } - if strings.EqualFold(headers[key], expected) { - t.Errorf("mtime header incorrectly parsed: got %s, wanted %s", headers[key], expected) - continue - } - trailer := make([]byte, 100) - n, err := reader.Read(trailer) - if err != io.EOF || n != 0 { - t.Error("Buffer wasn't consumed") - } - } - badHeader := bytes.NewBuffer([]byte("3 somelongkey=")) - if _, err := parsePAX(badHeader); err != ErrHeader { - t.Fatal("Unexpected success when parsing bad header") - } -} - -func TestParsePAXTime(t *testing.T) { - // Some valid PAX time values - timestamps := map[string]time.Time{ - "1350244992.023960108": time.Unix(1350244992, 23960108), // The commoon case - "1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value - "1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value - "1350244992": time.Unix(1350244992, 0), // Low precision value - } - for input, expected := range timestamps { - ts, err := parsePAXTime(input) - if err != nil { - t.Fatal(err) - } - if !ts.Equal(expected) { - t.Fatalf("Time parsing failure %s %s", ts, expected) - } - } -} - -func TestMergePAX(t *testing.T) { - hdr := new(Header) - // Test a string, integer, and time based value. - headers := map[string]string{ - "path": "a/b/c", - "uid": "1000", - "mtime": "1350244992.023960108", - } - err := mergePAX(hdr, headers) - if err != nil { - t.Fatal(err) - } - want := &Header{ - Name: "a/b/c", - Uid: 1000, - ModTime: time.Unix(1350244992, 23960108), - } - if !reflect.DeepEqual(hdr, want) { - t.Errorf("incorrect merge: got %+v, want %+v", hdr, want) - } -} diff --git a/vendor/src/github.com/dotcloud/tar/stat_atim.go b/vendor/src/github.com/dotcloud/tar/stat_atim.go deleted file mode 100644 index 6029b08712..0000000000 --- a/vendor/src/github.com/dotcloud/tar/stat_atim.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux openbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atim.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctim.Unix()) -} diff --git a/vendor/src/github.com/dotcloud/tar/stat_atimespec.go b/vendor/src/github.com/dotcloud/tar/stat_atimespec.go deleted file mode 100644 index 6f17dbe307..0000000000 --- a/vendor/src/github.com/dotcloud/tar/stat_atimespec.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin freebsd netbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atimespec.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctimespec.Unix()) -} diff --git a/vendor/src/github.com/dotcloud/tar/stat_unix.go b/vendor/src/github.com/dotcloud/tar/stat_unix.go deleted file mode 100644 index 92bc924242..0000000000 --- a/vendor/src/github.com/dotcloud/tar/stat_unix.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux darwin freebsd openbsd netbsd - -package tar - -import ( - "os" - "syscall" -) - -func init() { - sysStat = statUnix -} - -func statUnix(fi os.FileInfo, h *Header) error { - sys, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return nil - } - h.Uid = int(sys.Uid) - h.Gid = int(sys.Gid) - // TODO(bradfitz): populate username & group. os/user - // doesn't cache LookupId lookups, and lacks group - // lookup functions. - h.AccessTime = statAtime(sys) - h.ChangeTime = statCtime(sys) - // TODO(bradfitz): major/minor device numbers? - return nil -} diff --git a/vendor/src/github.com/dotcloud/tar/tar_test.go b/vendor/src/github.com/dotcloud/tar/tar_test.go deleted file mode 100644 index dd6310313a..0000000000 --- a/vendor/src/github.com/dotcloud/tar/tar_test.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "io/ioutil" - "os" - "reflect" - "testing" - "time" -) - -func TestFileInfoHeader(t *testing.T) { - fi, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - h, err := FileInfoHeader(fi, "") - if err != nil { - t.Fatalf("FileInfoHeader: %v", err) - } - if g, e := h.Name, "small.txt"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e { - t.Errorf("Mode = %#o; want %#o", g, e) - } - if g, e := h.Size, int64(5); g != e { - t.Errorf("Size = %v; want %v", g, e) - } - if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { - t.Errorf("ModTime = %v; want %v", g, e) - } -} - -func TestFileInfoHeaderDir(t *testing.T) { - fi, err := os.Stat("testdata") - if err != nil { - t.Fatal(err) - } - h, err := FileInfoHeader(fi, "") - if err != nil { - t.Fatalf("FileInfoHeader: %v", err) - } - if g, e := h.Name, "testdata/"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - // Ignoring c_ISGID for golang.org/issue/4867 - if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e { - t.Errorf("Mode = %#o; want %#o", g, e) - } - if g, e := h.Size, int64(0); g != e { - t.Errorf("Size = %v; want %v", g, e) - } - if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { - t.Errorf("ModTime = %v; want %v", g, e) - } -} - -func TestFileInfoHeaderSymlink(t *testing.T) { - h, err := FileInfoHeader(symlink{}, "some-target") - if err != nil { - t.Fatal(err) - } - if g, e := h.Name, "some-symlink"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - if g, e := h.Linkname, "some-target"; g != e { - t.Errorf("Linkname = %q; want %q", g, e) - } -} - -type symlink struct{} - -func (symlink) Name() string { return "some-symlink" } -func (symlink) Size() int64 { return 0 } -func (symlink) Mode() os.FileMode { return os.ModeSymlink } -func (symlink) ModTime() time.Time { return time.Time{} } -func (symlink) IsDir() bool { return false } -func (symlink) Sys() interface{} { return nil } - -func TestRoundTrip(t *testing.T) { - data := []byte("some file contents") - - var b bytes.Buffer - tw := NewWriter(&b) - hdr := &Header{ - Name: "file.txt", - Uid: 1 << 21, // too big for 8 octal digits - Size: int64(len(data)), - ModTime: time.Now(), - } - // tar only supports second precision. - hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) - if err := tw.WriteHeader(hdr); err != nil { - t.Fatalf("tw.WriteHeader: %v", err) - } - if _, err := tw.Write(data); err != nil { - t.Fatalf("tw.Write: %v", err) - } - if err := tw.Close(); err != nil { - t.Fatalf("tw.Close: %v", err) - } - - // Read it back. - tr := NewReader(&b) - rHdr, err := tr.Next() - if err != nil { - t.Fatalf("tr.Next: %v", err) - } - if !reflect.DeepEqual(rHdr, hdr) { - t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr) - } - rData, err := ioutil.ReadAll(tr) - if err != nil { - t.Fatalf("Read: %v", err) - } - if !bytes.Equal(rData, data) { - t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data) - } -} - -type headerRoundTripTest struct { - h *Header - fm os.FileMode -} - -func TestHeaderRoundTrip(t *testing.T) { - golden := []headerRoundTripTest{ - // regular file. - { - h: &Header{ - Name: "test.txt", - Mode: 0644 | c_ISREG, - Size: 12, - ModTime: time.Unix(1360600916, 0), - Typeflag: TypeReg, - }, - fm: 0644, - }, - // hard link. - { - h: &Header{ - Name: "hard.txt", - Mode: 0644 | c_ISLNK, - Size: 0, - ModTime: time.Unix(1360600916, 0), - Typeflag: TypeLink, - }, - fm: 0644 | os.ModeSymlink, - }, - // symbolic link. - { - h: &Header{ - Name: "link.txt", - Mode: 0777 | c_ISLNK, - Size: 0, - ModTime: time.Unix(1360600852, 0), - Typeflag: TypeSymlink, - }, - fm: 0777 | os.ModeSymlink, - }, - // character device node. - { - h: &Header{ - Name: "dev/null", - Mode: 0666 | c_ISCHR, - Size: 0, - ModTime: time.Unix(1360578951, 0), - Typeflag: TypeChar, - }, - fm: 0666 | os.ModeDevice | os.ModeCharDevice, - }, - // block device node. - { - h: &Header{ - Name: "dev/sda", - Mode: 0660 | c_ISBLK, - Size: 0, - ModTime: time.Unix(1360578954, 0), - Typeflag: TypeBlock, - }, - fm: 0660 | os.ModeDevice, - }, - // directory. - { - h: &Header{ - Name: "dir/", - Mode: 0755 | c_ISDIR, - Size: 0, - ModTime: time.Unix(1360601116, 0), - Typeflag: TypeDir, - }, - fm: 0755 | os.ModeDir, - }, - // fifo node. - { - h: &Header{ - Name: "dev/initctl", - Mode: 0600 | c_ISFIFO, - Size: 0, - ModTime: time.Unix(1360578949, 0), - Typeflag: TypeFifo, - }, - fm: 0600 | os.ModeNamedPipe, - }, - // setuid. - { - h: &Header{ - Name: "bin/su", - Mode: 0755 | c_ISREG | c_ISUID, - Size: 23232, - ModTime: time.Unix(1355405093, 0), - Typeflag: TypeReg, - }, - fm: 0755 | os.ModeSetuid, - }, - // setguid. - { - h: &Header{ - Name: "group.txt", - Mode: 0750 | c_ISREG | c_ISGID, - Size: 0, - ModTime: time.Unix(1360602346, 0), - Typeflag: TypeReg, - }, - fm: 0750 | os.ModeSetgid, - }, - // sticky. - { - h: &Header{ - Name: "sticky.txt", - Mode: 0600 | c_ISREG | c_ISVTX, - Size: 7, - ModTime: time.Unix(1360602540, 0), - Typeflag: TypeReg, - }, - fm: 0600 | os.ModeSticky, - }, - } - - for i, g := range golden { - fi := g.h.FileInfo() - h2, err := FileInfoHeader(fi, "") - if err != nil { - t.Error(err) - continue - } - if got, want := h2.Name, g.h.Name; got != want { - t.Errorf("i=%d: Name: got %v, want %v", i, got, want) - } - if got, want := h2.Size, g.h.Size; got != want { - t.Errorf("i=%d: Size: got %v, want %v", i, got, want) - } - if got, want := h2.Mode, g.h.Mode; got != want { - t.Errorf("i=%d: Mode: got %o, want %o", i, got, want) - } - if got, want := fi.Mode(), g.fm; got != want { - t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want) - } - if got, want := h2.ModTime, g.h.ModTime; got != want { - t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want) - } - if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h { - t.Errorf("i=%d: Sys didn't return original *Header", i) - } - } -} diff --git a/vendor/src/github.com/dotcloud/tar/testdata/gnu.tar b/vendor/src/github.com/dotcloud/tar/testdata/gnu.tar deleted file mode 100644 index fc899dc8dc2ad9952f5c5f67a0c76ca2d87249e9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3072 zcmeHH%L>9U5Ztq0(Jv@FdDKtv;8zq|ijXv5BIw^6DOh^2UK)_HbK1>@VQ0c5`qsHR zJrb1zXEcV16&lMRW}rdtKd=NSXg->JkvP|Esp4`g&CK_h+FMmo7oR?iU7RP&svn2t z!9Ke4)upeR_aRYKtT+(g`B!B>fS>t?p7IZ9V9LKWlK+)w+iY|SVQ_tY3I4Ddrx1w) M;($0H4*b6ZFWOBnBLDyZ diff --git a/vendor/src/github.com/dotcloud/tar/testdata/nil-uid.tar b/vendor/src/github.com/dotcloud/tar/testdata/nil-uid.tar deleted file mode 100644 index cc9cfaa33cc5de0a28b4183c1705d801f788c96a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1024 zcmWGAG%z(VGPcn33UJrU$xmmX0WbgpGcywmlR@HOU}(l*Xk=(?U}j`)Zf3?{U78BOoLqCLtvwr=Z5b$i&RT%Er#Y zO+ZjcSVUAHn~8MUp(~vBV+cg7LjpEKCCE6D7E@EwTcMv_>l+&bbg`j1Cv0A776ym5t@+ zSt9MDBFtXbKY&m4pMN0f`l~hhD>#q(-`x$5n+q@eEPmAevA;0XTM8XMYkTvSmQ-t5 zkihVw{(qQ#_JjT})&KMa&-FhG0c8or{CPvw|Jf69WL!B2Wa1KoKYcMW6^2fg(@@ia-%40!5$*6oDd81d2cr_`3;w E2V3|JA^-pY diff --git a/vendor/src/github.com/dotcloud/tar/testdata/small.txt b/vendor/src/github.com/dotcloud/tar/testdata/small.txt deleted file mode 100644 index b249bfc518..0000000000 --- a/vendor/src/github.com/dotcloud/tar/testdata/small.txt +++ /dev/null @@ -1 +0,0 @@ -Kilts \ No newline at end of file diff --git a/vendor/src/github.com/dotcloud/tar/testdata/small2.txt b/vendor/src/github.com/dotcloud/tar/testdata/small2.txt deleted file mode 100644 index 394ee3ecd0..0000000000 --- a/vendor/src/github.com/dotcloud/tar/testdata/small2.txt +++ /dev/null @@ -1 +0,0 @@ -Google.com diff --git a/vendor/src/github.com/dotcloud/tar/testdata/star.tar b/vendor/src/github.com/dotcloud/tar/testdata/star.tar deleted file mode 100644 index 59e2d4e604611eeac3e2a0f3d6f71d2623c50449..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3072 zcmeHHT?)e>4DRzz;R#BjQ;)ERouag*479>@u-$$NbG3!-WyoMlUh?xvNIv}HZD&jy zuA!-C5KZlY0Y@bP833Zfm_JQ2M2yBQ2dTK6K{>VDLBV=D{z>IvVF` zUD#xgUGh?F1AikeIW6NnOIrMRGU4UU`62nAWxyx>^STEhN#m{lQEc_E}GZPaA5N&Q|3Z@N=AbgM*P?o{a$k4#V#K6eR)QrKv#MIE( zgh9c8hHiozU0Pg{SOj!ZaYkZZDqIwk0aTWjhA9jefq29K;yD8YhMfGo^t{B}RQ&;E gz@3MSk&&8{lh1`qc2s;c1V%$(Gz3ONV7P_=0FTf|dgec!sXT^AK{$jKc@-kWdUe(&uh-&e0L+xARj zwLzm>LI~3|1sT#R&XkBIzWbfCPrYEK7fr^Q@7vXO;&pw$QCTT3-?&yO+jq(<{6qS`FS_vP zIBhMBjnmsnS~{|C9LMN8#r!W{zj5l&zcE?^U_t*||1zJ{zqInH{-Zy}2$O|c?WSFx zxn8RtM3-UpAJiW`Z@Zar#$ojz)NjtWBfnULUzD=jj5!>iG>O2k{o(=ZAg=$-urC7q zVm{n!{kK`S@p|Vk`q%aFg#nw)bMB-40yAj*%7=F37m@ziFINBH7pTSD@Cfil^^9T6 zxL-iu+Aq)#ev#CF(l2&S@A^eC<`;^e4{ZQ#s9$Y4r}$iP3;;e3V;a&MNN*s$f%FFc H(;N5+1FUK9 diff --git a/vendor/src/github.com/dotcloud/tar/testdata/writer-big.tar b/vendor/src/github.com/dotcloud/tar/testdata/writer-big.tar deleted file mode 100644 index 753e883cebf52ac1291f1b7bf1b7a37ae517b2d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmeIuF%E+;3tu-|!r}ytVByrmfae ipO37m$1T~NWs?FFpa2CZKmiI+fC3bt00k&;vcMnFf)<_t diff --git a/vendor/src/github.com/dotcloud/tar/testdata/writer.tar b/vendor/src/github.com/dotcloud/tar/testdata/writer.tar deleted file mode 100644 index e6d816ad0775d56d09242d6f5d1dbe56af310a32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3584 zcmeHIK@P$o5ajGDd_l9j6nKIMUt!cVjT92WM1L@81h#LhDgML6Bon)c?rO_kPgyt^3D0fH9$GJM`O*&4VCw= zv#H)UKC-TtzNwGuV$*%C{bm zsdIMLR{C5VZL^vBE!S4cfUeCYt@>GOiAt%sq7tp|_iN{x5cDreh9ME=K+wOCQm`$x j!znSk-v6Dy)}|V_!f*AilYjI7l|Jj-R%ReG@B;%+QQ}au diff --git a/vendor/src/github.com/dotcloud/tar/writer.go b/vendor/src/github.com/dotcloud/tar/writer.go deleted file mode 100644 index 549f1464c3..0000000000 --- a/vendor/src/github.com/dotcloud/tar/writer.go +++ /dev/null @@ -1,377 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -// TODO(dsymonds): -// - catch more errors (no first header, etc.) - -import ( - "bytes" - "errors" - "fmt" - "io" - "os" - "path" - "strconv" - "strings" - "time" -) - -var ( - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errNameTooLong = errors.New("archive/tar: name too long") - errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") -) - -// A Writer provides sequential writing of a tar archive in POSIX.1 format. -// A tar archive consists of a sequence of files. -// Call WriteHeader to begin a new file, and then call Write to supply that file's data, -// writing at most hdr.Size bytes in total. -type Writer struct { - w io.Writer - err error - nb int64 // number of unwritten bytes for current file entry - pad int64 // amount of padding to write after current file entry - closed bool - usedBinary bool // whether the binary numeric field extension was used - preferPax bool // use pax header instead of binary numeric header -} - -// NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } - -// Flush finishes writing the current file (optional). -func (tw *Writer) Flush() error { - if tw.nb > 0 { - tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb) - return tw.err - } - - n := tw.nb + tw.pad - for n > 0 && tw.err == nil { - nr := n - if nr > blockSize { - nr = blockSize - } - var nw int - nw, tw.err = tw.w.Write(zeroBlock[0:nr]) - n -= int64(nw) - } - tw.nb = 0 - tw.pad = 0 - return tw.err -} - -// Write s into b, terminating it with a NUL if there is room. -// If the value is too long for the field and allowPax is true add a paxheader record instead -func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) { - needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s) - if needsPaxHeader { - paxHeaders[paxKeyword] = s - return - } - if len(s) > len(b) { - if tw.err == nil { - tw.err = ErrFieldTooLong - } - return - } - ascii := toASCII(s) - copy(b, ascii) - if len(ascii) < len(b) { - b[len(ascii)] = 0 - } -} - -// Encode x as an octal ASCII string and write it into b with leading zeros. -func (tw *Writer) octal(b []byte, x int64) { - s := strconv.FormatInt(x, 8) - // leading zeros, but leave room for a NUL. - for len(s)+1 < len(b) { - s = "0" + s - } - tw.cString(b, s, false, paxNone, nil) -} - -// Write x into b, either as octal or as binary (GNUtar/star extension). -// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead -func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) { - // Try octal first. - s := strconv.FormatInt(x, 8) - if len(s) < len(b) { - tw.octal(b, x) - return - } - - // If it is too long for octal, and pax is preferred, use a pax header - if allowPax && tw.preferPax { - tw.octal(b, 0) - s := strconv.FormatInt(x, 10) - paxHeaders[paxKeyword] = s - return - } - - // Too big: use binary (big-endian). - tw.usedBinary = true - for i := len(b) - 1; x > 0 && i >= 0; i-- { - b[i] = byte(x) - x >>= 8 - } - b[0] |= 0x80 // highest bit indicates binary format -} - -var ( - minTime = time.Unix(0, 0) - // There is room for 11 octal digits (33 bits) of mtime. - maxTime = minTime.Add((1<<33 - 1) * time.Second) -) - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -func (tw *Writer) WriteHeader(hdr *Header) error { - return tw.writeHeader(hdr, true) -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -// As this method is called internally by writePax header to allow it to -// suppress writing the pax header. -func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { - if tw.closed { - return ErrWriteAfterClose - } - if tw.err == nil { - tw.Flush() - } - if tw.err != nil { - return tw.err - } - - // a map to hold pax header records, if any are needed - paxHeaders := make(map[string]string) - - // TODO(shanemhansen): we might want to use PAX headers for - // subsecond time resolution, but for now let's just capture - // too long fields or non ascii characters - - header := make([]byte, blockSize) - s := slicer(header) - - // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax - pathHeaderBytes := s.next(fileNameSize) - - tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders) - - // Handle out of range ModTime carefully. - var modTime int64 - if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { - modTime = hdr.ModTime.Unix() - } - - tw.octal(s.next(8), hdr.Mode) // 100:108 - tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116 - tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124 - tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136 - tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity - s.next(8) // chksum (148:156) - s.next(1)[0] = hdr.Typeflag // 156:157 - - tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders) - - copy(s.next(8), []byte("ustar\x0000")) // 257:265 - tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297 - tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329 - tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337 - tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345 - - // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax - prefixHeaderBytes := s.next(155) - tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix - - // Use the GNU magic instead of POSIX magic if we used any GNU extensions. - if tw.usedBinary { - copy(header[257:265], []byte("ustar \x00")) - } - - _, paxPathUsed := paxHeaders[paxPath] - // try to use a ustar header when only the name is too long - if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { - suffix := hdr.Name - prefix := "" - if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) { - var err error - prefix, suffix, err = tw.splitUSTARLongName(hdr.Name) - if err == nil { - // ok we can use a ustar long name instead of pax, now correct the fields - - // remove the path field from the pax header. this will suppress the pax header - delete(paxHeaders, paxPath) - - // update the path fields - tw.cString(pathHeaderBytes, suffix, false, paxNone, nil) - tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil) - - // Use the ustar magic if we used ustar long names. - if len(prefix) > 0 { - copy(header[257:265], []byte("ustar\000")) - } - } - } - } - - // The chksum field is terminated by a NUL and a space. - // This is different from the other octal fields. - chksum, _ := checksum(header) - tw.octal(header[148:155], chksum) - header[155] = ' ' - - if tw.err != nil { - // problem with header; probably integer too big for a field. - return tw.err - } - - if len(paxHeaders) > 0 { - if !allowPax { - return errInvalidHeader - } - if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { - return err - } - } - tw.nb = int64(hdr.Size) - tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize - - _, tw.err = tw.w.Write(header) - return tw.err -} - -// writeUSTARLongName splits a USTAR long name hdr.Name. -// name must be < 256 characters. errNameTooLong is returned -// if hdr.Name can't be split. The splitting heuristic -// is compatible with gnu tar. -func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) { - length := len(name) - if length > fileNamePrefixSize+1 { - length = fileNamePrefixSize + 1 - } else if name[length-1] == '/' { - length-- - } - i := strings.LastIndex(name[:length], "/") - // nlen contains the resulting length in the name field. - // plen contains the resulting length in the prefix field. - nlen := len(name) - i - 1 - plen := i - if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize { - err = errNameTooLong - return - } - prefix, suffix = name[:i], name[i+1:] - return -} - -// writePaxHeader writes an extended pax header to the -// archive. -func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { - // Prepare extended header - ext := new(Header) - ext.Typeflag = TypeXHeader - // Setting ModTime is required for reader parsing to - // succeed, and seems harmless enough. - ext.ModTime = hdr.ModTime - // The spec asks that we namespace our pseudo files - // with the current pid. - pid := os.Getpid() - dir, file := path.Split(hdr.Name) - fullName := path.Join(dir, - fmt.Sprintf("PaxHeaders.%d", pid), file) - - ascii := toASCII(fullName) - if len(ascii) > 100 { - ascii = ascii[:100] - } - ext.Name = ascii - // Construct the body - var buf bytes.Buffer - - for k, v := range paxHeaders { - fmt.Fprint(&buf, paxHeader(k+"="+v)) - } - - ext.Size = int64(len(buf.Bytes())) - if err := tw.writeHeader(ext, false); err != nil { - return err - } - if _, err := tw.Write(buf.Bytes()); err != nil { - return err - } - if err := tw.Flush(); err != nil { - return err - } - return nil -} - -// paxHeader formats a single pax record, prefixing it with the appropriate length -func paxHeader(msg string) string { - const padding = 2 // Extra padding for space and newline - size := len(msg) + padding - size += len(strconv.Itoa(size)) - record := fmt.Sprintf("%d %s\n", size, msg) - if len(record) != size { - // Final adjustment if adding size increased - // the number of digits in size - size = len(record) - record = fmt.Sprintf("%d %s\n", size, msg) - } - return record -} - -// Write writes to the current entry in the tar archive. -// Write returns the error ErrWriteTooLong if more than -// hdr.Size bytes are written after WriteHeader. -func (tw *Writer) Write(b []byte) (n int, err error) { - if tw.closed { - err = ErrWriteTooLong - return - } - overwrite := false - if int64(len(b)) > tw.nb { - b = b[0:tw.nb] - overwrite = true - } - n, err = tw.w.Write(b) - tw.nb -= int64(n) - if err == nil && overwrite { - err = ErrWriteTooLong - return - } - tw.err = err - return -} - -// Close closes the tar archive, flushing any unwritten -// data to the underlying writer. -func (tw *Writer) Close() error { - if tw.err != nil || tw.closed { - return tw.err - } - tw.Flush() - tw.closed = true - if tw.err != nil { - return tw.err - } - - // trailer: two zero blocks - for i := 0; i < 2; i++ { - _, tw.err = tw.w.Write(zeroBlock) - if tw.err != nil { - break - } - } - return tw.err -} diff --git a/vendor/src/github.com/dotcloud/tar/writer_test.go b/vendor/src/github.com/dotcloud/tar/writer_test.go deleted file mode 100644 index 30ebf977ac..0000000000 --- a/vendor/src/github.com/dotcloud/tar/writer_test.go +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "testing" - "testing/iotest" - "time" -) - -type writerTestEntry struct { - header *Header - contents string -} - -type writerTest struct { - file string // filename of expected output - entries []*writerTestEntry -} - -var writerTests = []*writerTest{ - // The writer test file was produced with this command: - // tar (GNU tar) 1.26 - // ln -s small.txt link.txt - // tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt - { - file: "testdata/writer.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1246508266, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - contents: "Kilts", - }, - { - header: &Header{ - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1245217492, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - contents: "Google.com\n", - }, - { - header: &Header{ - Name: "link.txt", - Mode: 0777, - Uid: 1000, - Gid: 1000, - Size: 0, - ModTime: time.Unix(1314603082, 0), - Typeflag: '2', - Linkname: "small.txt", - Uname: "strings", - Gname: "strings", - }, - // no contents - }, - }, - }, - // The truncated test file was produced using these commands: - // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt - // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar - { - file: "testdata/writer-big.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: "tmp/16gig.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 16 << 30, - ModTime: time.Unix(1254699560, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - // fake contents - contents: strings.Repeat("\x00", 4<<10), - }, - }, - }, - // This file was produced using gnu tar 1.17 - // gnutar -b 4 --format=ustar (longname/)*15 + file.txt - { - file: "testdata/ustar.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: strings.Repeat("longname/", 15) + "file.txt", - Mode: 0644, - Uid: 0765, - Gid: 024, - Size: 06, - ModTime: time.Unix(1360135598, 0), - Typeflag: '0', - Uname: "shane", - Gname: "staff", - }, - contents: "hello\n", - }, - }, - }, -} - -// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection. -func bytestr(offset int, b []byte) string { - const rowLen = 32 - s := fmt.Sprintf("%04x ", offset) - for _, ch := range b { - switch { - case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z': - s += fmt.Sprintf(" %c", ch) - default: - s += fmt.Sprintf(" %02x", ch) - } - } - return s -} - -// Render a pseudo-diff between two blocks of bytes. -func bytediff(a []byte, b []byte) string { - const rowLen = 32 - s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b)) - for offset := 0; len(a)+len(b) > 0; offset += rowLen { - na, nb := rowLen, rowLen - if na > len(a) { - na = len(a) - } - if nb > len(b) { - nb = len(b) - } - sa := bytestr(offset, a[0:na]) - sb := bytestr(offset, b[0:nb]) - if sa != sb { - s += fmt.Sprintf("-%v\n+%v\n", sa, sb) - } - a = a[na:] - b = b[nb:] - } - return s -} - -func TestWriter(t *testing.T) { -testLoop: - for i, test := range writerTests { - expected, err := ioutil.ReadFile(test.file) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - continue - } - - buf := new(bytes.Buffer) - tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB - big := false - for j, entry := range test.entries { - big = big || entry.header.Size > 1<<10 - if err := tw.WriteHeader(entry.header); err != nil { - t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err) - continue testLoop - } - if _, err := io.WriteString(tw, entry.contents); err != nil { - t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err) - continue testLoop - } - } - // Only interested in Close failures for the small tests. - if err := tw.Close(); err != nil && !big { - t.Errorf("test %d: Failed closing archive: %v", i, err) - continue testLoop - } - - actual := buf.Bytes() - if !bytes.Equal(expected, actual) { - t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v", - i, bytediff(expected, actual)) - } - if testing.Short() { // The second test is expensive. - break - } - } -} - -func TestPax(t *testing.T) { - // Create an archive with a large name - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - if err != nil { - t.Fatalf("os.Stat: %v", err) - } - // Force a PAX long name to be written - longName := strings.Repeat("ab", 100) - contents := strings.Repeat(" ", int(hdr.Size)) - hdr.Name = longName - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err = writer.Write([]byte(contents)); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != longName { - t.Fatal("Couldn't recover long file name") - } -} - -func TestPaxSymlink(t *testing.T) { - // Create an archive with a large linkname - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - hdr.Typeflag = TypeSymlink - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - // Force a PAX long linkname to be written - longLinkname := strings.Repeat("1234567890/1234567890", 10) - hdr.Linkname = longLinkname - - hdr.Size = 0 - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Linkname != longLinkname { - t.Fatal("Couldn't recover long link name") - } -} - -func TestPaxNonAscii(t *testing.T) { - // Create an archive with non ascii. These should trigger a pax header - // because pax headers have a defined utf-8 encoding. - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - - hdr, err := FileInfoHeader(fileinfo, "") - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - - // some sample data - chineseFilename := "文件名" - chineseGroupname := "組" - chineseUsername := "用戶名" - - hdr.Name = chineseFilename - hdr.Gname = chineseGroupname - hdr.Uname = chineseUsername - - contents := strings.Repeat(" ", int(hdr.Size)) - - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err = writer.Write([]byte(contents)); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != chineseFilename { - t.Fatal("Couldn't recover unicode name") - } - if hdr.Gname != chineseGroupname { - t.Fatal("Couldn't recover unicode group") - } - if hdr.Uname != chineseUsername { - t.Fatal("Couldn't recover unicode user") - } -} - -func TestPAXHeader(t *testing.T) { - medName := strings.Repeat("CD", 50) - longName := strings.Repeat("AB", 100) - paxTests := [][2]string{ - {paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"}, - {"a=b", "6 a=b\n"}, // Single digit length - {"a=names", "11 a=names\n"}, // Test case involving carries - {paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)}, - {paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}} - - for _, test := range paxTests { - key, expected := test[0], test[1] - if result := paxHeader(key); result != expected { - t.Fatalf("paxHeader: got %s, expected %s", result, expected) - } - } -} - -func TestUSTARLongName(t *testing.T) { - // Create an archive with a path that failed to split with USTAR extension in previous versions. - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - hdr.Typeflag = TypeDir - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - // Force a PAX long name to be written. The name was taken from a practical example - // that fails and replaced ever char through numbers to anonymize the sample. - longName := "/0000_0000000/00000-000000000/0000_0000000/00000-0000000000000/0000_0000000/00000-0000000-00000000/0000_0000000/00000000/0000_0000000/000/0000_0000000/00000000v00/0000_0000000/000000/0000_0000000/0000000/0000_0000000/00000y-00/0000/0000/00000000/0x000000/" - hdr.Name = longName - - hdr.Size = 0 - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != longName { - t.Fatal("Couldn't recover long name") - } -} From 45cea94a827bfcb77ef53e2e5046df51dc7c113c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 15:32:17 -0700 Subject: [PATCH 044/105] Unify hack/make/*test further by invoking hack/make/test directly from dyntest --- hack/make/dyntest | 60 +++-------------------------------------------- 1 file changed, 3 insertions(+), 57 deletions(-) diff --git a/hack/make/dyntest b/hack/make/dyntest index f7290c9557..cea3ba4e06 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -10,61 +10,7 @@ if [ ! -x "$INIT" ]; then false fi -TEXTRESET=$'\033[0m' # reset the foreground colour -RED=$'\033[31m' -GREEN=$'\033[32m' +export TEST_DOCKERINIT_PATH="$INIT" -# Run Docker's test suite, including sub-packages, and store their output as a bundle -# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. -# You can use this to select certain tests to run, eg. -# -# TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test -# -bundle_test() { - { - date - - export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION - - TESTS_FAILED=() - for test_dir in $(find_test_dirs); do - echo - - if ! LDFLAGS="$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" go_test_dir "$test_dir"; then - TESTS_FAILED+=("$test_dir") - echo - echo "${RED}Tests failed: $test_dir${TEXTRESET}" - sleep 1 # give it a second, so observers watching can take note - fi - done - - echo - echo - echo - - # if some tests fail, we want the bundlescript to fail, but we want to - # try running ALL the tests first, hence TESTS_FAILED - if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then - echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" - echo - false - else - echo "${GREEN}Test success${TEXTRESET}" - echo - true - fi - } 2>&1 | tee $DEST/test.log -} - - -# This helper function walks the current directory looking for directories -# holding Go test files, and prints their paths on standard output, one per -# line. -find_test_dirs() { - find -not \( \ - \( -wholename './vendor' -o -wholename './integration' \) \ - -prune \ - \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u -} - -bundle_test +LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" \ + source "$(dirname "$BASH_SOURCE")/test" From ca405786f468a18859684dc13d43a293d47b89bd Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 18:40:05 -0700 Subject: [PATCH 045/105] Unify dyntest/test and dynbinary/binary hack bundlescripts further by cross-invocation and keeping all the logic in one place, taking advantage of LDFLAGS_STATIC that is the only bit that gets replaced for dyntest/dynbinary --- hack/make/dynbinary | 6 ++++-- hack/make/dyntest | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hack/make/dynbinary b/hack/make/dynbinary index 96ce482674..100c00eed5 100644 --- a/hack/make/dynbinary +++ b/hack/make/dynbinary @@ -11,5 +11,7 @@ ln -sf dockerinit-$VERSION $DEST/dockerinit export DOCKER_INITSHA1="$(sha1sum $DEST/dockerinit-$VERSION | cut -d' ' -f1)" # exported so that "dyntest" can easily access it later without recalculating it -go build -o $DEST/docker-$VERSION -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS ./docker -echo "Created binary: $DEST/docker-$VERSION" +( + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" + source "$(dirname "$BASH_SOURCE")/binary" +) diff --git a/hack/make/dyntest b/hack/make/dyntest index cea3ba4e06..eb5c2b73ed 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -10,7 +10,8 @@ if [ ! -x "$INIT" ]; then false fi -export TEST_DOCKERINIT_PATH="$INIT" - -LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" \ +( + export TEST_DOCKERINIT_PATH="$INIT" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" source "$(dirname "$BASH_SOURCE")/test" +) From f0879a1e145f31569e9e4e61f429de858bb636ab Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 18:41:50 -0700 Subject: [PATCH 046/105] Add separate "test-integration" bundlescript (and corresponding dyntest-integration bundlescript) --- Makefile | 2 +- hack/make.sh | 2 ++ hack/make/dyntest-integration | 17 +++++++++++++++++ hack/make/test-integration | 11 +++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 hack/make/dyntest-integration create mode 100644 hack/make/test-integration diff --git a/Makefile b/Makefile index 1518fb1331..02a121dc50 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ doc: docker build -t docker-docs docs && docker run -p 8000:8000 docker-docs test: build - $(DOCKER_RUN_DOCKER) hack/make.sh test + $(DOCKER_RUN_DOCKER) hack/make.sh test test-integration shell: build $(DOCKER_RUN_DOCKER) bash diff --git a/hack/make.sh b/hack/make.sh index 23b77509b6..7b39f3161c 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -35,8 +35,10 @@ grep -q "$RESOLVCONF" /proc/mounts || { DEFAULT_BUNDLES=( binary test + test-integration dynbinary dyntest + dyntest-integration tgz ubuntu ) diff --git a/hack/make/dyntest-integration b/hack/make/dyntest-integration new file mode 100644 index 0000000000..0887c45be0 --- /dev/null +++ b/hack/make/dyntest-integration @@ -0,0 +1,17 @@ +#!/bin/bash + +DEST=$1 +INIT=$DEST/../dynbinary/dockerinit-$VERSION + +set -e + +if [ ! -x "$INIT" ]; then + echo >&2 'error: dynbinary must be run before dyntest-integration' + false +fi + +( + export TEST_DOCKERINIT_PATH="$INIT" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" + source "$(dirname "$BASH_SOURCE")/test-integration" +) diff --git a/hack/make/test-integration b/hack/make/test-integration new file mode 100644 index 0000000000..f1ab0b99c3 --- /dev/null +++ b/hack/make/test-integration @@ -0,0 +1,11 @@ +#!/bin/bash + +DEST=$1 + +set -e + +bundle_test_integration() { + LDFLAGS="$LDFLAGS $LDFLAGS_STATIC" go_test_dir ./integration +} + +bundle_test_integration 2>&1 | tee $DEST/test.log From e2ee5c71fc665170db6a3a487aebb4d7f42eed0b Mon Sep 17 00:00:00 2001 From: Isaac Dupree Date: Mon, 9 Dec 2013 12:25:20 -0500 Subject: [PATCH 047/105] Make it extra clear that the `docker` group is root-equivalent. --- docs/sources/use/basics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 12160312ba..0881bde233 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -67,7 +67,7 @@ daemon will make the ownership of the Unix socket read/writable by the *docker* group when the daemon starts. The ``docker`` daemon must always run as root, but if you run the ``docker`` client as a user in the *docker* group then you don't need to add ``sudo`` to all the -client commands. +client commands. Warning: the *docker* group is root-equivalent. **Example:** From 776bb43c9e97824a2cb05353c68ef5645f3cc112 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Mon, 9 Dec 2013 20:46:21 +0000 Subject: [PATCH 048/105] Prevent deletion of image if ANY container is depending on it; not just running containers --- server.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/server.go b/server.go index 49b25f7098..392dc81447 100644 --- a/server.go +++ b/server.go @@ -1554,22 +1554,20 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - // Prevent deletion if image is used by a running container + // Prevent deletion if image is used by a container for _, container := range srv.runtime.List() { - if container.State.IsRunning() { - parent, err := srv.runtime.repositories.LookupImage(container.Image) - if err != nil { - return nil, err - } + parent, err := srv.runtime.repositories.LookupImage(container.Image) + if err != nil { + return nil, err + } - if err := parent.WalkHistory(func(p *Image) error { - if img.ID == p.ID { - return fmt.Errorf("Conflict, cannot delete %s because the running container %s is using it", name, container.ID) - } - return nil - }); err != nil { - return nil, err + if err := parent.WalkHistory(func(p *Image) error { + if img.ID == p.ID { + return fmt.Errorf("Conflict, cannot delete %s because the container %s is using it", name, container.ID) } + return nil + }); err != nil { + return nil, err } } From 8291f00a0e656cf3bb52fb1dcdff7c266305f4d5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 9 Dec 2013 16:25:19 -0800 Subject: [PATCH 049/105] refactor and fix tests --- api.go | 20 +++++++++++--------- integration/buildfile_test.go | 6 +++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/api.go b/api.go index fb59f5efd3..bd8e53d655 100644 --- a/api.go +++ b/api.go @@ -905,15 +905,17 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if version < 1.3 { return fmt.Errorf("Multipart upload for build is no longer supported. Please upgrade your docker client.") } - remoteURL := r.FormValue("remote") - repoName := r.FormValue("t") - rawSuppressOutput := r.FormValue("q") - rawNoCache := r.FormValue("nocache") - rawRm := r.FormValue("rm") - repoName, tag := utils.ParseRepositoryTag(repoName) - - authEncoded := r.Header.Get("X-Registry-Auth") - authConfig := &auth.AuthConfig{} + var ( + remoteURL = r.FormValue("remote") + repoName = r.FormValue("t") + rawSuppressOutput = r.FormValue("q") + rawNoCache = r.FormValue("nocache") + rawRm = r.FormValue("rm") + authEncoded = r.Header.Get("X-Registry-Auth") + authConfig = &auth.AuthConfig{} + tag string + ) + repoName, tag = utils.ParseRepositoryTag(repoName) if authEncoded != "" { authJson := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authEncoded)) if err := json.NewDecoder(authJson).Decode(authConfig); err != nil { diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 242bf9f412..4d15031d30 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -266,7 +266,7 @@ func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, u } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, ioutil.Discard, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, ioutil.Discard, utils.NewStreamFormatter(false), nil) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { return nil, err @@ -516,7 +516,7 @@ func TestForbiddenContextPath(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false), nil) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { @@ -562,7 +562,7 @@ func TestBuildADDFileNotFound(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false), nil) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { From c9cedb4c0484c3e1879495ab5c02088ae738763e Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Mon, 9 Dec 2013 16:47:19 -0800 Subject: [PATCH 050/105] Some minor cleanup of the Links use document --- docs/sources/use/working_with_links_names.rst | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/sources/use/working_with_links_names.rst b/docs/sources/use/working_with_links_names.rst index e6ab9395f8..6283a865f8 100644 --- a/docs/sources/use/working_with_links_names.rst +++ b/docs/sources/use/working_with_links_names.rst @@ -1,6 +1,6 @@ :title: Working with Links and Names :description: How to create and use links and names -:keywords: Examples, Usage, links, docker, documentation, examples, names, name, container naming +:keywords: Examples, Usage, links, linking, docker, documentation, examples, names, name, container naming .. _working_with_links_names: @@ -40,7 +40,7 @@ Links: service discovery for docker Links allow containers to discover and securely communicate with each other by using the flag ``-link name:alias``. Inter-container communication can be disabled with the daemon -flag ``-icc=false``. With this flag set to false, Container A cannot access Container B +flag ``-icc=false``. With this flag set to ``false``, Container A cannot access Container B unless explicitly allowed via a link. This is a huge win for securing your containers. When two containers are linked together Docker creates a parent child relationship between the containers. The parent container will be able to access information via @@ -52,28 +52,29 @@ a secure tunnel for the parent to access. If a database container only exposes p then the linked container will only be allowed to access port 8080 and nothing else if inter-container communication is set to false. +For example, there is an image called ``crosbymichael/redis`` that exposes the +port 6379 and starts the Redis server. Let's name the container as ``redis`` +based on that image and run it as daemon. + .. code-block:: bash - # Example: there is an image called crosbymichael/redis that exposes the port 6379 and starts redis-server. - # Let's name the container as "redis" based on that image and run it as daemon. $ sudo docker run -d -name redis crosbymichael/redis -We can issue all the commands that you would expect using the name "redis"; start, stop, +We can issue all the commands that you would expect using the name ``redis``; start, stop, attach, using the name for our container. The name also allows us to link other containers into this one. Next, we can start a new web application that has a dependency on Redis and apply a link to connect both containers. If you noticed when running our Redis server we did not use -the -p flag to publish the Redis port to the host system. Redis exposed port 6379 and +the ``-p`` flag to publish the Redis port to the host system. Redis exposed port 6379 and this is all we need to establish a link. .. code-block:: bash - # Linking the redis container as a child $ sudo docker run -t -i -link redis:db -name webapp ubuntu bash -When you specified -link redis:db you are telling docker to link the container named redis -into this new container with the alias db. Environment variables are prefixed with the alias +When you specified ``-link redis:db`` you are telling Docker to link the container named ``redis`` +into this new container with the alias ``db``. Environment variables are prefixed with the alias so that the parent container can access network and environment information from the containers that are linked into it. @@ -103,11 +104,12 @@ about the child container. Accessing the network information along with the environment of the child container allows us to easily connect to the Redis service on the specific IP and port in the environment. -Running ``docker ps`` shows the 2 containers, and the webapp/db alias name for the redis container. +Running ``docker ps`` shows the 2 containers, and the ``webapp/db`` alias name for the redis container. .. code-block:: bash $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 4c01db0b339c ubuntu:12.04 bash 17 seconds ago Up 16 seconds webapp - d7886598dbe2 crosbymichael/redis:latest /redis-server --dir 33 minutes ago Up 33 minutes 6379/tcp redis,webapp/db + 4c01db0b339c ubuntu:12.04 bash 17 seconds ago Up 16 seconds webapp + d7886598dbe2 crosbymichael/redis:latest /redis-server --dir 33 minutes ago Up 33 minutes 6379/tcp redis,webapp/db + From ac1093b83aee38243b0d6df3dd8aabc52062b9d2 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 9 Dec 2013 10:43:58 -0800 Subject: [PATCH 051/105] fix docker images -tree and docker images -viz --- commands.go | 105 ++++++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/commands.go b/commands.go index 48c0241b17..935dd33913 100644 --- a/commands.go +++ b/commands.go @@ -1102,33 +1102,9 @@ func (cli *DockerCli) CmdImages(args ...string) error { return nil } - if *flViz { - body, _, err := cli.call("GET", "/images/json?all=1", nil) - if err != nil { - return err - } + filter := cmd.Arg(0) - var outs []APIImages - err = json.Unmarshal(body, &outs) - if err != nil { - return err - } - - fmt.Fprintf(cli.out, "digraph docker {\n") - - for _, image := range outs { - if image.ParentId == "" { - fmt.Fprintf(cli.out, " base -> \"%s\" [style=invis]\n", utils.TruncateID(image.ID)) - } else { - fmt.Fprintf(cli.out, " \"%s\" -> \"%s\"\n", utils.TruncateID(image.ParentId), utils.TruncateID(image.ID)) - } - if image.RepoTags[0] != ":" { - fmt.Fprintf(cli.out, " \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", utils.TruncateID(image.ID), utils.TruncateID(image.ID), strings.Join(image.RepoTags, "\\n")) - } - } - - fmt.Fprintf(cli.out, " base [style=invisible]\n}\n") - } else if *flTree { + if *flViz || *flTree { body, _, err := cli.call("GET", "/images/json?all=1", nil) if err != nil { return err @@ -1140,8 +1116,8 @@ func (cli *DockerCli) CmdImages(args ...string) error { } var ( - startImageArg = cmd.Arg(0) - startImage APIImages + printNode func(cli *DockerCli, noTrunc bool, image APIImages, prefix string) + startImage APIImages roots []APIImages byParent = make(map[string][]APIImages) @@ -1158,28 +1134,38 @@ func (cli *DockerCli) CmdImages(args ...string) error { } } - if startImageArg != "" { - if startImageArg == image.ID || startImageArg == utils.TruncateID(image.ID) { + if filter != "" { + if filter == image.ID || filter == utils.TruncateID(image.ID) { startImage = image } for _, repotag := range image.RepoTags { - if repotag == startImageArg { + if repotag == filter { startImage = image } } } } - if startImageArg != "" { - WalkTree(cli, noTrunc, []APIImages{startImage}, byParent, "") + if *flViz { + fmt.Fprintf(cli.out, "digraph docker {\n") + printNode = (*DockerCli).printVizNode } else { - WalkTree(cli, noTrunc, roots, byParent, "") + printNode = (*DockerCli).printTreeNode + } + + if startImage.ID != "" { + cli.WalkTree(*noTrunc, &[]APIImages{startImage}, byParent, "", printNode) + } else if filter == "" { + cli.WalkTree(*noTrunc, &roots, byParent, "", printNode) + } + if *flViz { + fmt.Fprintf(cli.out, " base [style=invisible]\n}\n") } } else { v := url.Values{} if cmd.NArg() == 1 { - v.Set("filter", cmd.Arg(0)) + v.Set("filter", filter) } if *all { v.Set("all", "1") @@ -1225,35 +1211,58 @@ func (cli *DockerCli) CmdImages(args ...string) error { return nil } -func WalkTree(cli *DockerCli, noTrunc *bool, images []APIImages, byParent map[string][]APIImages, prefix string) { - if len(images) > 1 { - length := len(images) - for index, image := range images { +func (cli *DockerCli) WalkTree(noTrunc bool, images *[]APIImages, byParent map[string][]APIImages, prefix string, printNode func(cli *DockerCli, noTrunc bool, image APIImages, prefix string)) { + length := len(*images) + if length > 1 { + for index, image := range *images { if index+1 == length { - PrintTreeNode(cli, noTrunc, image, prefix+"└─") + printNode(cli, noTrunc, image, prefix+"└─") if subimages, exists := byParent[image.ID]; exists { - WalkTree(cli, noTrunc, subimages, byParent, prefix+" ") + cli.WalkTree(noTrunc, &subimages, byParent, prefix+" ", printNode) } } else { - PrintTreeNode(cli, noTrunc, image, prefix+"|─") + printNode(cli, noTrunc, image, prefix+"|─") if subimages, exists := byParent[image.ID]; exists { - WalkTree(cli, noTrunc, subimages, byParent, prefix+"| ") + cli.WalkTree(noTrunc, &subimages, byParent, prefix+"| ", printNode) } } } } else { - for _, image := range images { - PrintTreeNode(cli, noTrunc, image, prefix+"└─") + for _, image := range *images { + printNode(cli, noTrunc, image, prefix+"└─") if subimages, exists := byParent[image.ID]; exists { - WalkTree(cli, noTrunc, subimages, byParent, prefix+" ") + cli.WalkTree(noTrunc, &subimages, byParent, prefix+" ", printNode) } } } } -func PrintTreeNode(cli *DockerCli, noTrunc *bool, image APIImages, prefix string) { +func (cli *DockerCli) printVizNode(noTrunc bool, image APIImages, prefix string) { + var ( + imageID string + parentID string + ) + if noTrunc { + imageID = image.ID + parentID = image.ParentId + } else { + imageID = utils.TruncateID(image.ID) + parentID = utils.TruncateID(image.ParentId) + } + if image.ParentId == "" { + fmt.Fprintf(cli.out, " base -> \"%s\" [style=invis]\n", imageID) + } else { + fmt.Fprintf(cli.out, " \"%s\" -> \"%s\"\n", parentID, imageID) + } + if image.RepoTags[0] != ":" { + fmt.Fprintf(cli.out, " \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", + imageID, imageID, strings.Join(image.RepoTags, "\\n")) + } +} + +func (cli *DockerCli) printTreeNode(noTrunc bool, image APIImages, prefix string) { var imageID string - if *noTrunc { + if noTrunc { imageID = image.ID } else { imageID = utils.TruncateID(image.ID) From 9c1e9a5157b3ef7627f4b735c2b8b6d97f26dd53 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Fri, 6 Dec 2013 15:26:57 -0800 Subject: [PATCH 052/105] Fix #1229. Update titles, fix some wrapping. Make the Ambassador container explicit. Apply Sven's suggestions. --- .../use/ambassador_pattern_linking.rst | 6 +- docs/sources/use/baseimages.rst | 4 +- docs/sources/use/basics.rst | 6 +- docs/sources/use/builder.rst | 8 +- docs/sources/use/host_integration.rst | 6 +- docs/sources/use/index.rst | 2 +- docs/sources/use/networking.rst | 68 +++++++++------- docs/sources/use/port_redirection.rst | 6 +- docs/sources/use/working_with_links_names.rst | 81 ++++++++++--------- docs/sources/use/working_with_volumes.rst | 6 +- docs/sources/use/workingwithrepository.rst | 6 +- 11 files changed, 107 insertions(+), 92 deletions(-) diff --git a/docs/sources/use/ambassador_pattern_linking.rst b/docs/sources/use/ambassador_pattern_linking.rst index 095324cb16..d6d307783e 100644 --- a/docs/sources/use/ambassador_pattern_linking.rst +++ b/docs/sources/use/ambassador_pattern_linking.rst @@ -1,11 +1,11 @@ -:title: Ambassador pattern linking +:title: Link via an Ambassador Container :description: Using the Ambassador pattern to abstract (network) services :keywords: Examples, Usage, links, docker, documentation, examples, names, name, container naming .. _ambassador_pattern_linking: -Ambassador pattern linking -========================== +Link via an Ambassador Container +================================ Rather than hardcoding network links between a service consumer and provider, Docker encourages service portability. diff --git a/docs/sources/use/baseimages.rst b/docs/sources/use/baseimages.rst index f1f5d106e0..51a51e2f93 100644 --- a/docs/sources/use/baseimages.rst +++ b/docs/sources/use/baseimages.rst @@ -1,10 +1,10 @@ -:title: Base Image Creation +:title: Create a Base Image :description: How to create base images :keywords: Examples, Usage, base image, docker, documentation, examples .. _base_image_creation: -Base Image Creation +Create a Base Image =================== So you want to create your own :ref:`base_image_def`? Great! diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 0881bde233..6b2c588817 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -1,10 +1,10 @@ -:title: Basic Commands +:title: Learn Basic Commands :description: Common usage and commands :keywords: Examples, Usage, basic commands, docker, documentation, examples -The Basics -========== +Learn Basic Commands +==================== Starting Docker --------------- diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index 2035c276dd..10140313a8 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -1,12 +1,12 @@ -:title: Dockerfiles for Images +:title: Build Images (Dockerfile Reference) :description: Dockerfiles use a simple DSL which allows you to automate the steps you would normally manually take to create an image. :keywords: builder, docker, Dockerfile, automation, image creation .. _dockerbuilder: -====================== -Dockerfiles for Images -====================== +=================================== +Build Images (Dockerfile Reference) +=================================== **Docker can act as a builder** and read instructions from a text ``Dockerfile`` to automate the steps you would otherwise take manually diff --git a/docs/sources/use/host_integration.rst b/docs/sources/use/host_integration.rst index a9f9c1e753..fb70195ffd 100644 --- a/docs/sources/use/host_integration.rst +++ b/docs/sources/use/host_integration.rst @@ -1,11 +1,11 @@ -:title: Host Integration +:title: Automatically Start Containers :description: How to generate scripts for upstart, systemd, etc. :keywords: systemd, upstart, supervisor, docker, documentation, host integration -Host Integration -================ +Automatically Start Containers +============================== You can use your Docker containers with process managers like ``upstart``, ``systemd`` and ``supervisor``. diff --git a/docs/sources/use/index.rst b/docs/sources/use/index.rst index ef95a517a7..7bcd1dd81e 100644 --- a/docs/sources/use/index.rst +++ b/docs/sources/use/index.rst @@ -17,9 +17,9 @@ Contents: workingwithrepository baseimages port_redirection - puppet networking host_integration working_with_volumes working_with_links_names ambassador_pattern_linking + puppet diff --git a/docs/sources/use/networking.rst b/docs/sources/use/networking.rst index c37035114e..0e81440e03 100644 --- a/docs/sources/use/networking.rst +++ b/docs/sources/use/networking.rst @@ -1,14 +1,14 @@ -:title: Docker networking +:title: Configure Networking :description: Docker networking :keywords: network, networking, bridge, docker, documentation -Networking -========== +Configure Networking +==================== -Docker uses Linux bridge capabilities to provide network connectivity -to containers. The ``docker0`` bridge interface is managed by Docker itself -for this purpose. Thus, when the Docker daemon starts it : +Docker uses Linux bridge capabilities to provide network connectivity +to containers. The ``docker0`` bridge interface is managed by Docker +itself for this purpose. Thus, when the Docker daemon starts it : - creates the ``docker0`` bridge if not present - searches for an IP address range which doesn't overlap with an existing route @@ -30,11 +30,12 @@ for this purpose. Thus, when the Docker daemon starts it : -At runtime, a :ref:`specific kind of virtual interface` is -given to each containers which is then bonded to the ``docker0`` bridge. -Each containers also receives a dedicated IP address from the same range -as ``docker0``. The ``docker0`` IP address is then used as the default -gateway for the containers. +At runtime, a :ref:`specific kind of virtual +interface` is given to each containers which is then +bonded to the ``docker0`` bridge. Each containers also receives a +dedicated IP address from the same range as ``docker0``. The +``docker0`` IP address is then used as the default gateway for the +containers. .. code-block:: bash @@ -47,18 +48,19 @@ gateway for the containers. docker0 8000.fef213db5a66 no vethQCDY1N -Above, ``docker0`` acts as a bridge for the ``vethQCDY1N`` interface which is dedicated -to the 52f811c5d3d6 container. +Above, ``docker0`` acts as a bridge for the ``vethQCDY1N`` interface +which is dedicated to the 52f811c5d3d6 container. How to use a specific IP address range --------------------------------------- -Docker will try hard to find an IP range which is not used by the host. -Even if it works for most cases, it's not bullet-proof and sometimes you need -to have more control over the IP addressing scheme. -For this purpose, Docker allows you to manage the ``docker0`` bridge or -your own one using the ``-b=`` parameter. +Docker will try hard to find an IP range which is not used by the +host. Even if it works for most cases, it's not bullet-proof and +sometimes you need to have more control over the IP addressing scheme. + +For this purpose, Docker allows you to manage the ``docker0`` bridge +or your own one using the ``-b=`` parameter. In this scenario: @@ -115,14 +117,15 @@ In this scenario: Container intercommunication ------------------------------- -Containers can communicate with each other according to the ``icc`` parameter -value of the Docker daemon. + +Containers can communicate with each other according to the ``icc`` +parameter value of the Docker daemon. - The default, ``-icc=true`` allows containers to communicate with each other. - ``-icc=false`` means containers are isolated from each other. -Under the hood, ``iptables`` is used by Docker to either accept or drop communication -between containers. +Under the hood, ``iptables`` is used by Docker to either accept or +drop communication between containers. .. _vethxxxx-device: @@ -131,17 +134,20 @@ What's about the vethXXXX device? ----------------------------------- Well. Things get complicated here. -The ``vethXXXX`` interface is the host side of a point-to-point link between the -host and the corresponding container, the other side of the link being -materialized by the container's ``eth0`` interface. This pair (host ``vethXXX`` and -container ``eth0``) are connected like a tube. Everything that comes in one side will -come out the other side. +The ``vethXXXX`` interface is the host side of a point-to-point link +between the host and the corresponding container, the other side of +the link being materialized by the container's ``eth0`` +interface. This pair (host ``vethXXX`` and container ``eth0``) are +connected like a tube. Everything that comes in one side will come out +the other side. -All the plumbing is delegated to Linux network capabilities (check the ip link -command) and the namespaces infrastructure. +All the plumbing is delegated to Linux network capabilities (check the +ip link command) and the namespaces infrastructure. I want more ------------ -Jérôme Petazzoni has create ``pipework`` to connect together containers in -arbitrarily complex scenarios : https://github.com/jpetazzo/pipework + +Jérôme Petazzoni has create ``pipework`` to connect together +containers in arbitrarily complex scenarios : +https://github.com/jpetazzo/pipework diff --git a/docs/sources/use/port_redirection.rst b/docs/sources/use/port_redirection.rst index adce1cba61..b35d27a3db 100644 --- a/docs/sources/use/port_redirection.rst +++ b/docs/sources/use/port_redirection.rst @@ -1,12 +1,12 @@ -:title: Port redirection +:title: Redirect Ports :description: usage about port redirection :keywords: Usage, basic port, docker, documentation, examples .. _port_redirection: -Port redirection -================ +Redirect Ports +============== Interacting with a service is commonly done through a connection to a port. When this service runs inside a container, one can connect to diff --git a/docs/sources/use/working_with_links_names.rst b/docs/sources/use/working_with_links_names.rst index 6283a865f8..8c654cbdbc 100644 --- a/docs/sources/use/working_with_links_names.rst +++ b/docs/sources/use/working_with_links_names.rst @@ -1,15 +1,16 @@ -:title: Working with Links and Names -:description: How to create and use links and names +:title: Link Containers +:description: How to create and use both links and names :keywords: Examples, Usage, links, linking, docker, documentation, examples, names, name, container naming .. _working_with_links_names: -Working with Links and Names -============================ +Link Containers +=============== -From version 0.6.5 you are now able to ``name`` a container and ``link`` it to another -container by referring to its name. This will create a parent -> child relationship -where the parent container can see selected information about its child. +From version 0.6.5 you are now able to ``name`` a container and +``link`` it to another container by referring to its name. This will +create a parent -> child relationship where the parent container can +see selected information about its child. .. _run_name: @@ -18,8 +19,9 @@ Container Naming .. versionadded:: v0.6.5 -You can now name your container by using the ``-name`` flag. If no name is provided, Docker -will automatically generate a name. You can see this name using the ``docker ps`` command. +You can now name your container by using the ``-name`` flag. If no +name is provided, Docker will automatically generate a name. You can +see this name using the ``docker ps`` command. .. code-block:: bash @@ -38,18 +40,21 @@ Links: service discovery for docker .. versionadded:: v0.6.5 -Links allow containers to discover and securely communicate with each other by using the -flag ``-link name:alias``. Inter-container communication can be disabled with the daemon -flag ``-icc=false``. With this flag set to ``false``, Container A cannot access Container B -unless explicitly allowed via a link. This is a huge win for securing your containers. -When two containers are linked together Docker creates a parent child relationship -between the containers. The parent container will be able to access information via -environment variables of the child such as name, exposed ports, IP and other selected -environment variables. +Links allow containers to discover and securely communicate with each +other by using the flag ``-link name:alias``. Inter-container +communication can be disabled with the daemon flag +``-icc=false``. With this flag set to ``false``, Container A cannot +access Container B unless explicitly allowed via a link. This is a +huge win for securing your containers. When two containers are linked +together Docker creates a parent child relationship between the +containers. The parent container will be able to access information +via environment variables of the child such as name, exposed ports, IP +and other selected environment variables. -When linking two containers Docker will use the exposed ports of the container to create -a secure tunnel for the parent to access. If a database container only exposes port 8080 -then the linked container will only be allowed to access port 8080 and nothing else if +When linking two containers Docker will use the exposed ports of the +container to create a secure tunnel for the parent to access. If a +database container only exposes port 8080 then the linked container +will only be allowed to access port 8080 and nothing else if inter-container communication is set to false. For example, there is an image called ``crosbymichael/redis`` that exposes the @@ -60,26 +65,28 @@ based on that image and run it as daemon. $ sudo docker run -d -name redis crosbymichael/redis -We can issue all the commands that you would expect using the name ``redis``; start, stop, -attach, using the name for our container. The name also allows us to link other containers -into this one. +We can issue all the commands that you would expect using the name +``redis``; start, stop, attach, using the name for our container. The +name also allows us to link other containers into this one. -Next, we can start a new web application that has a dependency on Redis and apply a link -to connect both containers. If you noticed when running our Redis server we did not use -the ``-p`` flag to publish the Redis port to the host system. Redis exposed port 6379 and -this is all we need to establish a link. +Next, we can start a new web application that has a dependency on +Redis and apply a link to connect both containers. If you noticed when +running our Redis server we did not use the ``-p`` flag to publish the +Redis port to the host system. Redis exposed port 6379 and this is all +we need to establish a link. .. code-block:: bash $ sudo docker run -t -i -link redis:db -name webapp ubuntu bash -When you specified ``-link redis:db`` you are telling Docker to link the container named ``redis`` -into this new container with the alias ``db``. Environment variables are prefixed with the alias -so that the parent container can access network and environment information from the containers -that are linked into it. +When you specified ``-link redis:db`` you are telling Docker to link +the container named ``redis`` into this new container with the alias +``db``. Environment variables are prefixed with the alias so that the +parent container can access network and environment information from +the containers that are linked into it. -If we inspect the environment variables of the second container, we would see all the information -about the child container. +If we inspect the environment variables of the second container, we +would see all the information about the child container. .. code-block:: bash @@ -101,10 +108,12 @@ about the child container. _=/usr/bin/env root@4c01db0b339c:/# -Accessing the network information along with the environment of the child container allows -us to easily connect to the Redis service on the specific IP and port in the environment. +Accessing the network information along with the environment of the +child container allows us to easily connect to the Redis service on +the specific IP and port in the environment. -Running ``docker ps`` shows the 2 containers, and the ``webapp/db`` alias name for the redis container. +Running ``docker ps`` shows the 2 containers, and the ``webapp/db`` +alias name for the redis container. .. code-block:: bash diff --git a/docs/sources/use/working_with_volumes.rst b/docs/sources/use/working_with_volumes.rst index 9156e574d0..b5ef991138 100644 --- a/docs/sources/use/working_with_volumes.rst +++ b/docs/sources/use/working_with_volumes.rst @@ -1,11 +1,11 @@ -:title: Working with Volumes +:title: Share Directories via Volumes :description: How to create and share volumes :keywords: Examples, Usage, volume, docker, documentation, examples .. _volume_def: -Data Volume -=========== +Share Directories via Volumes +============================= .. versionadded:: v0.3.0 Data volumes have been available since version 1 of the diff --git a/docs/sources/use/workingwithrepository.rst b/docs/sources/use/workingwithrepository.rst index 3797f6ab52..38062556cb 100644 --- a/docs/sources/use/workingwithrepository.rst +++ b/docs/sources/use/workingwithrepository.rst @@ -1,11 +1,11 @@ -:title: Working With Repositories +:title: Share Images via Repositories :description: Repositories allow users to share images. :keywords: repo, repositories, usage, pull image, push image, image, documentation .. _working_with_the_repository: -Working with Repositories -========================= +Share Images via Repositories +============================= A *repository* is a hosted collection of tagged :ref:`images ` that together create the file system for a container. The From c618a906a4a50b4cc76e9230f971d737a59b8c27 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 9 Dec 2013 16:59:42 -0800 Subject: [PATCH 053/105] fix size in -tree --- commands.go | 2 +- integration/commands_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 935dd33913..3546c9a41b 100644 --- a/commands.go +++ b/commands.go @@ -1268,7 +1268,7 @@ func (cli *DockerCli) printTreeNode(noTrunc bool, image APIImages, prefix string imageID = utils.TruncateID(image.ID) } - fmt.Fprintf(cli.out, "%s%s Size: %s (virtual %s)", prefix, imageID, utils.HumanSize(image.Size), utils.HumanSize(image.VirtualSize)) + fmt.Fprintf(cli.out, "%s%s Virtual Size: %s", prefix, imageID, utils.HumanSize(image.VirtualSize)) if image.RepoTags[0] != ":" { fmt.Fprintf(cli.out, " Tags: %s\n", strings.Join(image.RepoTags, ", ")) } else { diff --git a/integration/commands_test.go b/integration/commands_test.go index 16e2b3fb02..7bba37a51b 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -867,11 +867,11 @@ func TestImagesTree(t *testing.T) { } cmdOutput := string(cmdOutputBytes) regexpStrings := []string{ - fmt.Sprintf("└─%s Size: (\\d+.\\d+ MB) \\(virtual \\d+.\\d+ MB\\) Tags: %s:latest", unitTestImageIDShort, unitTestImageName), + fmt.Sprintf("└─%s Virtual Size: \\d+.\\d+ MB Tags: %s:latest", unitTestImageIDShort, unitTestImageName), "(?m) └─[0-9a-f]+.*", "(?m) └─[0-9a-f]+.*", "(?m) └─[0-9a-f]+.*", - fmt.Sprintf("(?m)^ └─%s Size: \\d+ B \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)), + fmt.Sprintf("(?m)^ └─%s Virtual Size: \\d+.\\d+ MB Tags: test:latest", utils.TruncateID(image.ID)), } compiledRegexps := []*regexp.Regexp{} From 78b85220be021bff022e40037693f42575e8ba8d Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 10 Dec 2013 10:59:32 -0700 Subject: [PATCH 054/105] Update mkimage-debootstrap with even more tweaks for keeping images tiny by more aggressively removing cache files and by not downloading apt-cache Translations files --- contrib/mkimage-debootstrap.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/mkimage-debootstrap.sh b/contrib/mkimage-debootstrap.sh index a078801b90..f9992d6e3c 100755 --- a/contrib/mkimage-debootstrap.sh +++ b/contrib/mkimage-debootstrap.sh @@ -142,14 +142,22 @@ if [ -z "$strictDebootstrap" ]; then # this forces dpkg not to call sync() after package extraction and speeds up install # the benefit is huge on spinning disks, and the penalty is nonexistent on SSD or decent server virtualization echo 'force-unsafe-io' | sudo tee etc/dpkg/dpkg.cfg.d/02apt-speedup > /dev/null - # we want to effectively run "apt-get clean" after every install to keep images small - echo 'DPkg::Post-Invoke {"/bin/rm -f /var/cache/apt/archives/*.deb || true";};' | sudo tee etc/apt/apt.conf.d/no-cache > /dev/null + # we want to effectively run "apt-get clean" after every install to keep images small (see output of "apt-get clean -s" for context) + { + aptGetClean='rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true' + echo 'DPkg::Post-Invoke { "'$aptGetClean'"; };' + echo 'APT::Update::Post-Invoke { "'$aptGetClean'"; };' + echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' + } | sudo tee etc/apt/apt.conf.d/no-cache > /dev/null + # and remove the translations, too + echo 'Acquire::Languages "none";' | sudo tee etc/apt/apt.conf.d/no-languages > /dev/null # helpful undo lines for each the above tweaks (for lack of a better home to keep track of them): # rm /usr/sbin/policy-rc.d # rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl # rm /etc/dpkg/dpkg.cfg.d/02apt-speedup # rm /etc/apt/apt.conf.d/no-cache + # rm /etc/apt/apt.conf.d/no-languages if [ -z "$skipDetection" ]; then # see also rudimentary platform detection in hack/install.sh From 761184df5295b72c5d1ab4549dcddb54977da487 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Tue, 10 Dec 2013 12:49:53 -0600 Subject: [PATCH 055/105] don't open bind mounted files/dirs to get Stat, use os.Lstat --- container.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/container.go b/container.go index bb0723eada..36330909ce 100644 --- a/container.go +++ b/container.go @@ -801,15 +801,10 @@ func (container *Container) createVolumes() error { if strings.ToLower(bindMap.Mode) == "rw" { srcRW = true } - if file, err := os.Open(bindMap.SrcPath); err != nil { + if stat, err := os.Lstat(bindMap.SrcPath); err != nil { return err } else { - defer file.Close() - if stat, err := file.Stat(); err != nil { - return err - } else { - volIsDir = stat.IsDir() - } + volIsDir = stat.IsDir() } // Otherwise create an directory in $ROOT/volumes/ and use that } else { From 95f061b408eafbf981450d3fdb26ebc3188cba4a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 10 Dec 2013 10:57:16 -0800 Subject: [PATCH 056/105] update docker push to use [====> ] --- graph.go | 2 +- server.go | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/graph.go b/graph.go index 82128bfe15..da236d3716 100644 --- a/graph.go +++ b/graph.go @@ -219,7 +219,7 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, if err != nil { return nil, err } - return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf, true, "", "Buffering to disk"), tmp) + return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf, false, utils.TruncateID(id), "Buffering to disk"), tmp) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/server.go b/server.go index 5f08f08dff..3f1308f041 100644 --- a/server.go +++ b/server.go @@ -1093,7 +1093,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName return nil } - out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", utils.TruncateID(elem.ID), ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } @@ -1103,13 +1103,13 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName if err := pushTags(); err != nil { return err } - out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatProgress(utils.TruncateID(elem.ID), "Image already pushed, skipping", nil)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { if err := pushTags(); err != nil { return err } - out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatProgress(utils.TruncateID(elem.ID), "Image already pushed, skipping", nil)) continue } checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf) @@ -1139,7 +1139,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err != nil { return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) } - out.Write(sf.FormatStatus("", "Pushing %s", imgID)) + out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pushing", nil)) imgData := ®istry.ImgData{ ID: imgID, @@ -1148,7 +1148,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID)) + out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) return "", nil } return "", err @@ -1161,14 +1161,11 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, defer os.RemoveAll(layerData.Name()) // Send the layer - checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf, false, "", "Pushing"), ep, token, jsonRaw) + checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf, false, utils.TruncateID(imgData.ID), "Pushing"), ep, token, jsonRaw) if err != nil { return "", err } imgData.Checksum = checksum - - out.Write(sf.FormatStatus("", "")) - // Send the checksum if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { return "", err From b98d51dddb081e4bf4f54cbc9c48ef123c2254d0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 10 Dec 2013 15:37:03 -0800 Subject: [PATCH 057/105] revert 'firstErr' --- engine/streams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/streams.go b/engine/streams.go index 697407f1f4..ff7049074a 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -89,7 +89,7 @@ func (o *Output) Write(p []byte) (n int, err error) { firstErr = err } } - return len(p), err + return len(p), firstErr } // Close unregisters all destinations and waits for all background From 27646c445951b47a728675764d7ef2b198436156 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 11 Dec 2013 10:14:56 +1000 Subject: [PATCH 058/105] make docs is more consistent --- Makefile | 4 ++-- docs/README.md | 2 +- docs/sources/contributing/devenvironment.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1518fb1331..b9f7158dbd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary build default doc shell test +.PHONY: all binary build default docs shell test DOCKER_RUN_DOCKER := docker run -rm -i -t -privileged -e TESTFLAGS -v $(CURDIR)/bundles:/go/src/github.com/dotcloud/docker/bundles docker @@ -10,7 +10,7 @@ all: build binary: build $(DOCKER_RUN_DOCKER) hack/make.sh binary -doc: +docs: docker build -t docker-docs docs && docker run -p 8000:8000 docker-docs test: build diff --git a/docs/README.md b/docs/README.md index 3f4725a583..768cabdbb1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,7 +57,7 @@ docs inside the container, even starting a simple HTTP server on port 8000 so that you can connect and see your changes. In the ``docker`` source directory, run: - ```make doc``` + ```make docs``` This is the equivalent to ``make clean server`` since each container starts clean. diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index c152ff9ce5..1c2ea1a35d 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -129,7 +129,7 @@ to it, you can build the documentation and then serve it by: .. code-block:: bash - sudo make doc + sudo make docs # when its done, you can point your browser to http://yourdockerhost:8000 # type Ctrl-C to exit From be137350018608bf4a1fd01ce956f166c39988c4 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 11 Dec 2013 11:12:11 +1000 Subject: [PATCH 059/105] associate swapping the built docker binary with building the binary, rather than a note in building the docs --- docs/sources/contributing/devenvironment.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index c152ff9ce5..d9649c6178 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -44,7 +44,8 @@ This following command will build a development environment using the Dockerfile sudo make build -If the build is successful, congratulations! You have produced a clean build of docker, neatly encapsulated in a standard build environment. +If the build is successful, congratulations! You have produced a clean build of +docker, neatly encapsulated in a standard build environment. Step 4: Build the Docker Binary @@ -58,6 +59,19 @@ To create the Docker binary, run this command: This will create the Docker binary in ``./bundles/-dev/binary/`` +Using your built Docker binary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The binary is available outside the container in the directory +``./bundles/-dev/binary/``. You can swap your host docker executable +with this binary for live testing - for example, on ubuntu: + +.. code-block:: bash + + sudo service docker stop ; sudo cp $(which docker) $(which docker)_ ; sudo cp ./bundles/-dev/binary/docker--dev $(which docker);sudo service docker start + +.. note:: Its safer to run the tests below before swapping your hosts docker binary. + Step 5: Run the Tests --------------------- @@ -134,9 +148,6 @@ to it, you can build the documentation and then serve it by: # type Ctrl-C to exit -.. note:: The binary is available outside the container in the directory ``./bundles/-dev/binary/``. You can swap your host docker executable with this binary for live testing - for example, on ubuntu: ``sudo service docker stop ; sudo cp $(which docker) $(which docker)_ ; sudo cp ./bundles/-dev/binary/docker--dev $(which docker);sudo service docker start``. - - **Need More Help?** If you need more help then hop on to the `#docker-dev IRC channel `_ or post a message on the `Docker developer mailinglist `_. From d878632b2548247fc9e4c2da1b2520e4c7b9420a Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 11 Dec 2013 12:07:07 +1000 Subject: [PATCH 060/105] add a direct example for changing the cmd that is run --- docs/sources/commandline/cli.rst | 33 +++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 5cbd28c951..585c0f92aa 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -225,8 +225,10 @@ by using the ``git://`` schema. -run="": Configuration to be applied when the image is launched with `docker run`. (ex: -run='{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}') -Simple commit of an existing container -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _cli_commit_examples: + +Commit an existing container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: bash @@ -240,10 +242,33 @@ Simple commit of an existing container REPOSITORY TAG ID CREATED VIRTUAL SIZE SvenDowideit/testimage version3 f5283438590d 16 seconds ago 335.7 MB +Change the command that a container runs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you have an application container running just a service and you need +to make a quick change (run bash?) and then change it back. + +In this example, we run a container with ``ls`` and then change the image to +run ``ls /etc``. + +.. code-block:: bash + + $ docker run -t -name test ubuntu ls + bin boot dev etc home lib lib64 media mnt opt proc root run sbin selinux srv sys tmp usr var + $ docker commit -run='{"Cmd": ["ls","/etc"]}' test test2 + 933d16de9e70005304c1717b5c6f2f39d6fd50752834c6f34a155c70790011eb + $ docker run -t test2 + adduser.conf gshadow login.defs rc0.d + alternatives gshadow- logrotate.d rc1.d + apt host.conf lsb-base rc2.d + ... Full -run example ................. +The ``-run`` JSON hash changes the ``Config`` section when running ``docker inspect CONTAINERID`` +or ``config`` when running ``docker inspect IMAGEID``. + (multiline is ok within a single quote ``'``) :: @@ -943,6 +968,8 @@ the specified image, and then ``'starts'`` it using the specified command. That is, ``'docker run'`` is equivalent to the API ``/containers/create`` then ``/containers/(id)/start``. +``docker run`` can be used in combination with ``docker commit`` to :ref:`change the command that a container runs `. + Known Issues (run -volumes-from) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1085,7 +1112,7 @@ in the same mode (rw or ro) as the reference container. :: - Usage: docker start [OPTIONS] NAME + Usage: docker start [OPTIONS] CONTAINER Start a stopped container From 7edd1f6bad752dd89eeaa3690e219dfa748afa9c Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 11 Dec 2013 15:54:34 +1000 Subject: [PATCH 061/105] add example for docker rmi, and explain the need to remove all references (tags) to and image before its garbage collected :) --- docs/sources/commandline/cli.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 5cbd28c951..95bf6dce91 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -900,6 +900,38 @@ containers will not be deleted. Usage: docker rmi IMAGE [IMAGE...] Remove one or more images + +Removing tagged images +~~~~~~~~~~~~~~~~~~~~~~ + +Images can be removed either by their short or long ID's, or their image names. +If an image has more than one name, each of them needs to be removed before the +image is removed. + +.. code-block:: bash + + $ sudo docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + test1 latest fd484f19954f 23 seconds ago 7 B (virtual 4.964 MB) + test latest fd484f19954f 23 seconds ago 7 B (virtual 4.964 MB) + test2 latest fd484f19954f 23 seconds ago 7 B (virtual 4.964 MB) + + $ sudo docker rmi fd484f19954f + Error: Conflict, fd484f19954f wasn't deleted + 2013/12/11 05:47:16 Error: failed to remove one or more images + + $ sudo docker rmi test1 + Untagged: fd484f19954f4920da7ff372b5067f5b7ddb2fd3830cecd17b96ea9e286ba5b8 + $ sudo docker rmi test2 + Untagged: fd484f19954f4920da7ff372b5067f5b7ddb2fd3830cecd17b96ea9e286ba5b8 + + $ sudo docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + test1 latest fd484f19954f 23 seconds ago 7 B (virtual 4.964 MB) + $ sudo docker rmi test + Untagged: fd484f19954f4920da7ff372b5067f5b7ddb2fd3830cecd17b96ea9e286ba5b8 + Deleted: fd484f19954f4920da7ff372b5067f5b7ddb2fd3830cecd17b96ea9e286ba5b8 + .. _cli_run: From af21908493e570a1399bbd462803398ef34e56bc Mon Sep 17 00:00:00 2001 From: Quentin Brossard Date: Wed, 11 Dec 2013 13:15:27 +0100 Subject: [PATCH 062/105] Corrected typo (resdis -> redis) --- docs/sources/use/ambassador_pattern_linking.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/use/ambassador_pattern_linking.rst b/docs/sources/use/ambassador_pattern_linking.rst index d6d307783e..e7cdbd7c96 100644 --- a/docs/sources/use/ambassador_pattern_linking.rst +++ b/docs/sources/use/ambassador_pattern_linking.rst @@ -27,7 +27,7 @@ you can add ambassadors (consumer) --> (redis-ambassador) ---network---> (redis-ambassador) --> (redis) -When you need to rewire your consumer to talk to a different resdis server, you +When you need to rewire your consumer to talk to a different redis server, you can just restart the ``redis-ambassador`` container that the consumer is connected to. This pattern also allows you to transparently move the redis server to a different From 81fc368a6d70ef369bd16a3c090c6910851faacf Mon Sep 17 00:00:00 2001 From: Richo Healey Date: Wed, 11 Dec 2013 11:27:36 -0800 Subject: [PATCH 063/105] Use https to get the install script --- docs/sources/installation/ubuntulinux.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 73e7c3a80e..00400a94bc 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -77,7 +77,7 @@ First add the Docker repository key to your local keychain. You can use the sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" Add the Docker repository to your apt sources list, update and install the -``lxc-docker`` package. +``lxc-docker`` package. *You may receive a warning that the package isn't trusted. Answer yes to continue installation.* @@ -95,7 +95,7 @@ continue installation.* .. code-block:: bash - curl -s http://get.docker.io/ubuntu/ | sudo sh + curl -s https://get.docker.io/ubuntu/ | sudo sh Now verify that the installation has worked by downloading the ``ubuntu`` image and launching a container. From bef8de93194a442c21481f0b7e72f2fac781f799 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Nov 2013 07:37:03 +0000 Subject: [PATCH 064/105] Engine: integer job status, improved stream API * Jobs return an integer status instead of a string * Status convention mimics unix process execution: 0=success, 1=generic error, 127="no such command" * Stdout and Stderr support multiple thread-safe data receivers and ring buffer filtering --- engine/job.go | 2 +- engine/streams.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/engine/job.go b/engine/job.go index 478f06cd60..465086af91 100644 --- a/engine/job.go +++ b/engine/job.go @@ -24,7 +24,7 @@ type Job struct { Eng *Engine Name string Args []string - env *Env + env *Env Stdout *Output Stderr *Output Stdin *Input diff --git a/engine/streams.go b/engine/streams.go index ee26fe5f1e..7cd4a60cf7 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -168,7 +168,7 @@ func Tail(src io.Reader, n int, dst *[]string) { // AddEnv starts a new goroutine which will decode all subsequent data // as a stream of json-encoded objects, and point `dst` to the last // decoded object. -// The result `env` can be queried using the type-neutral Env interface. +// The result `env` can be queried using the type-neutral Env interface. // It is not safe to query `env` until the Output is closed. func (o *Output) AddEnv() (dst *Env, err error) { src, err := o.AddPipe() @@ -185,9 +185,8 @@ func (o *Output) AddEnv() (dst *Env, err error) { if err != nil { return } - *dst= *env + *dst = *env } }() return dst, nil } - From dbe1915fee1beb034459fb875928c96e27ecbfce Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 17 Nov 2013 03:00:16 +0000 Subject: [PATCH 065/105] Engine: new command 'stop' gracefully stops a container. --- api.go | 13 +++++-------- integration/server_test.go | 5 ++--- server.go | 32 +++++++++++++++++++++++++++----- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/api.go b/api.go index 58fa310240..1555923018 100644 --- a/api.go +++ b/api.go @@ -683,17 +683,14 @@ func postContainersStop(srv *Server, version float64, w http.ResponseWriter, r * if err := parseForm(r); err != nil { return err } - t, err := strconv.Atoi(r.Form.Get("t")) - if err != nil || t < 0 { - t = 10 - } - if vars == nil { return fmt.Errorf("Missing parameter") } - name := vars["name"] - - if err := srv.ContainerStop(name, t); err != nil { + job := srv.Eng.Job("stop", vars["name"]) + if t := r.Form.Get("t"); t != "" { + job.Args = append(job.Args, t) + } + if err := job.Run(); err != nil { return err } w.WriteHeader(http.StatusNoContent) diff --git a/integration/server_test.go b/integration/server_test.go index c1499e26cc..9eab174b99 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -128,8 +128,7 @@ func TestCreateRmVolumes(t *testing.T) { t.Fatal(err) } - err = srv.ContainerStop(id, 1) - if err != nil { + if err := eng.Job("stop", id, "1").Run(); err != nil { t.Fatal(err) } @@ -187,7 +186,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerStop(id, 15); err != nil { + if err := eng.Job("stop", id, "15").Run(); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index c32cfe23ed..01cdee0473 100644 --- a/server.go +++ b/server.go @@ -79,6 +79,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("stop", srv.ContainerStop); err != nil { + job.Error(err) + return engine.StatusErr + } if err := job.Eng.Register("start", srv.ContainerStart); err != nil { job.Error(err) return engine.StatusErr @@ -1713,16 +1717,34 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { return engine.StatusOK } -func (srv *Server) ContainerStop(name string, t int) error { +func (srv *Server) ContainerStop(job *engine.Job) engine.Status { + if len(job.Args) < 1 { + job.Errorf("Not enough arguments. Usage: %s CONTAINER TIMEOUT\n", job.Name) + return engine.StatusErr + } + name := job.Args[0] + var t uint64 + if len(job.Args) == 2 { + var err error + t, err = strconv.ParseUint(job.Args[1], 10, 32) + if err != nil { + job.Errorf("Invalid delay format: %s. Please provide an integer number of seconds.\n", job.Args[1]) + return engine.StatusErr + } + } else { + t = 10 + } if container := srv.runtime.Get(name); container != nil { - if err := container.Stop(t); err != nil { - return fmt.Errorf("Cannot stop container %s: %s", name, err) + if err := container.Stop(int(t)); err != nil { + job.Errorf("Cannot stop container %s: %s\n", name, err) + return engine.StatusErr } srv.LogEvent("stop", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { - return fmt.Errorf("No such container: %s", name) + job.Errorf("No such container: %s\n", name) + return engine.StatusErr } - return nil + return engine.StatusOK } func (srv *Server) ContainerWait(job *engine.Job) engine.Status { From eec48f93a36873b8255c1c533b0d2bfbd742f9fa Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 14:34:51 -0600 Subject: [PATCH 066/105] rhel docs update Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index 443d5a4ffc..cfbddf740f 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -1,6 +1,6 @@ -:title: Requirements and Installation on Red Hat Enterprise Linux / CentOS +:title: Requirements and Installation on Red Hat Enterprise Linux :description: Please note this project is currently under heavy development. It should not be used in production. -:keywords: Docker, Docker documentation, requirements, linux, rhel, centos +:keywords: Docker, Docker documentation, requirements, linux, rhel .. _rhel: @@ -11,7 +11,7 @@ Red Hat Enterprise Linux / CentOS .. include:: install_unofficial.inc -Docker is available for **RHEL/CentOS 6**. +Docker is available for **RHEL**. Please note that this package is part of a `Extra Packages for Enterprise Linux (EPEL)`_, a community effort to create and maintain additional packages for RHEL distribution. @@ -20,21 +20,15 @@ Please note that due to the current Docker limitations Docker is able to run onl Installation ------------ -1. Firstly, let's make sure our RHEL host is up-to-date. +1. First, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. -.. code-block:: bash - - sudo yum -y upgrade - -2. Next you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. - -3. Next let's install the ``docker-io`` package which will install Docker on our host. +2. Next let's install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash sudo yum -y install docker-io -4. Now it's installed lets start the Docker daemon. +3. Now it's installed lets start the Docker daemon. .. code-block:: bash @@ -46,11 +40,11 @@ If we want Docker to start at boot we should also: sudo chkconfig docker on -5. Now let's verify that Docker is working. +4. Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t ubuntu /bin/bash + sudo docker run -i -t mattdm/fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. From 31961ccd94bc2d0941d3062591206a58b3b451a1 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 14:36:12 -0600 Subject: [PATCH 067/105] rhel page only for rhel Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index cfbddf740f..afc3548702 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -4,8 +4,8 @@ .. _rhel: -Red Hat Enterprise Linux / CentOS -================================= +Red Hat Enterprise Linux +======================== .. include:: install_header.inc From 39cc8a32b1dcf6dcb9462fbc2768ae035e8095dc Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 11 Dec 2013 14:13:55 -0700 Subject: [PATCH 068/105] Fix James's github handle in docs/MAINTAINERS --- docs/MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index 5f83c635d8..d782e43c5e 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,4 +1,4 @@ Andy Rothfusz (@metalivedev) Ken Cochrane (@kencochrane) -James Turnbull (@jamesturnbull) +James Turnbull (@jamtur01) Sven Dowideit (@SvenDowideit) From 42c38bf34dff7a5af4c75646b7252d3ecbdd334a Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 15:59:35 -0600 Subject: [PATCH 069/105] rhel description update Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index afc3548702..a4484afaef 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -11,7 +11,9 @@ Red Hat Enterprise Linux .. include:: install_unofficial.inc -Docker is available for **RHEL**. +Docker is available for **RHEL** on EPEL. These instructions should work for +both RHEL and CentOS. They will likely work with +other binary compatible EL6 version as well, but they haven't been tested. Please note that this package is part of a `Extra Packages for Enterprise Linux (EPEL)`_, a community effort to create and maintain additional packages for RHEL distribution. From db3019d50b7a38dd52d8ad959017c63b04f11462 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 16:00:46 -0600 Subject: [PATCH 070/105] rhel page keywords update Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index a4484afaef..3204af133a 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -1,6 +1,6 @@ :title: Requirements and Installation on Red Hat Enterprise Linux :description: Please note this project is currently under heavy development. It should not be used in production. -:keywords: Docker, Docker documentation, requirements, linux, rhel +:keywords: Docker, Docker documentation, requirements, linux, rhel, centos .. _rhel: From d534e1c3a1e00786d63e9dbeffceb6ab7f51c92b Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 16:20:54 -0600 Subject: [PATCH 071/105] some typo corrections Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index 3204af133a..2385b614ce 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -12,25 +12,34 @@ Red Hat Enterprise Linux .. include:: install_unofficial.inc Docker is available for **RHEL** on EPEL. These instructions should work for -both RHEL and CentOS. They will likely work with -other binary compatible EL6 version as well, but they haven't been tested. +both RHEL and CentOS. They will likely work with other binary compatible EL6 +distributions as well, but the haven't been tested. -Please note that this package is part of a `Extra Packages for Enterprise Linux (EPEL)`_, a community effort to create and maintain additional packages for RHEL distribution. +Please note that this package is part of `Extra Packages for Enterprise +Linux (EPEL)`_, a community effort to create and maintain additional packages +for the RHEL distribution. -Please note that due to the current Docker limitations Docker is able to run only on the **64 bit** architecture. +Please note that due to the current Docker limitations, Docker is able to run +only on the **64 bit** architecture. Installation ------------ -1. First, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. +1. Firstly, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. -2. Next let's install the ``docker-io`` package which will install Docker on our host. +2. Next, let's install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash sudo yum -y install docker-io -3. Now it's installed lets start the Docker daemon. +To keep docker up to date: + +.. code-block:: bash + + sudo yum -y update docker-io + +3. Now that it's installed, lets start the Docker daemon. .. code-block:: bash From 44984602c7c7460454531c22ee4dfab2a34fa334 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 16:36:14 -0600 Subject: [PATCH 072/105] more typo corrections Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index 2385b614ce..f1169a0c79 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -12,14 +12,14 @@ Red Hat Enterprise Linux .. include:: install_unofficial.inc Docker is available for **RHEL** on EPEL. These instructions should work for -both RHEL and CentOS. They will likely work with other binary compatible EL6 -distributions as well, but the haven't been tested. +both RHEL and CentOS. They will likely work for other binary compatible EL6 +distributions as well, but they haven't been tested. Please note that this package is part of `Extra Packages for Enterprise Linux (EPEL)`_, a community effort to create and maintain additional packages for the RHEL distribution. -Please note that due to the current Docker limitations, Docker is able to run +Also note that due to the current Docker limitations, Docker is able to run only on the **64 bit** architecture. Installation From 6ba456ff87cfe2687db7df896e2983636ca9fe1e Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 11 Dec 2013 15:36:50 -0800 Subject: [PATCH 073/105] move t from arg to env --- api.go | 4 +--- integration/server_test.go | 8 ++++++-- server.go | 15 ++++----------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/api.go b/api.go index 1555923018..f72e1f5afd 100644 --- a/api.go +++ b/api.go @@ -687,9 +687,7 @@ func postContainersStop(srv *Server, version float64, w http.ResponseWriter, r * return fmt.Errorf("Missing parameter") } job := srv.Eng.Job("stop", vars["name"]) - if t := r.Form.Get("t"); t != "" { - job.Args = append(job.Args, t) - } + job.Setenv("t", r.Form.Get("t")) if err := job.Run(); err != nil { return err } diff --git a/integration/server_test.go b/integration/server_test.go index 9eab174b99..e469abb610 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -128,7 +128,9 @@ func TestCreateRmVolumes(t *testing.T) { t.Fatal(err) } - if err := eng.Job("stop", id, "1").Run(); err != nil { + job = eng.Job("stop", id) + job.SetenvInt("t", 1) + if err := job.Run(); err != nil { t.Fatal(err) } @@ -186,7 +188,9 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := eng.Job("stop", id, "15").Run(); err != nil { + job = eng.Job("stop", id) + job.SetenvInt("t", 15) + if err := job.Run(); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index 01cdee0473..ed0c4ac1fa 100644 --- a/server.go +++ b/server.go @@ -1718,20 +1718,13 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { } func (srv *Server) ContainerStop(job *engine.Job) engine.Status { - if len(job.Args) < 1 { - job.Errorf("Not enough arguments. Usage: %s CONTAINER TIMEOUT\n", job.Name) + if len(job.Args) != 1 { + job.Errorf("Usage: %s CONTAINER\n", job.Name) return engine.StatusErr } name := job.Args[0] - var t uint64 - if len(job.Args) == 2 { - var err error - t, err = strconv.ParseUint(job.Args[1], 10, 32) - if err != nil { - job.Errorf("Invalid delay format: %s. Please provide an integer number of seconds.\n", job.Args[1]) - return engine.StatusErr - } - } else { + t := job.GetenvInt("t") + if t == -1 { t = 10 } if container := srv.runtime.Get(name); container != nil { From e8ec3dba7b1eb9254a04c852ffe7d98d10d0d476 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 18:21:52 -0600 Subject: [PATCH 074/105] remove step numbers, keep consistent with fedora Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index f1169a0c79..9429e74d61 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -25,9 +25,10 @@ only on the **64 bit** architecture. Installation ------------ -1. Firstly, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. +Firstly, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. -2. Next, let's install the ``docker-io`` package which will install Docker on our host. + +Next, let's install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash @@ -39,7 +40,7 @@ To keep docker up to date: sudo yum -y update docker-io -3. Now that it's installed, lets start the Docker daemon. +Now that it's installed, let's start the Docker daemon. .. code-block:: bash @@ -51,7 +52,7 @@ If we want Docker to start at boot we should also: sudo chkconfig docker on -4. Now let's verify that Docker is working. +Now let's verify that Docker is working. .. code-block:: bash From 8874f2aef9ed12574160394fd7a6a254c0ef0526 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 11 Dec 2013 18:26:17 -0600 Subject: [PATCH 075/105] keeping rhel page sorta in sync with fedora Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/rhel.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index 9429e74d61..993d329df7 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -34,7 +34,7 @@ Next, let's install the ``docker-io`` package which will install Docker on our h sudo yum -y install docker-io -To keep docker up to date: +To update the ``docker-io`` package .. code-block:: bash @@ -46,7 +46,7 @@ Now that it's installed, let's start the Docker daemon. sudo service docker start -If we want Docker to start at boot we should also: +If we want Docker to start at boot, we should also: .. code-block:: bash From f396c42cad2b23dae5e3296c18fd6190f210c022 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 11 Dec 2013 16:31:02 -0800 Subject: [PATCH 076/105] Fix volumes on the host by following symlinks in a scope --- container.go | 29 ++++++++------- utils/fs.go | 47 ++++++++++++++++++++++++ utils/fs_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++ utils/testdata/fs/a/d | 1 + utils/testdata/fs/a/e | 1 + utils/testdata/fs/a/f | 1 + 6 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 utils/fs_test.go create mode 120000 utils/testdata/fs/a/d create mode 120000 utils/testdata/fs/a/e create mode 120000 utils/testdata/fs/a/f diff --git a/container.go b/container.go index 36330909ce..5959ec7600 100644 --- a/container.go +++ b/container.go @@ -774,14 +774,14 @@ func (container *Container) getBindMap() (map[string]BindMap, error) { } binds[path.Clean(dst)] = bindMap } - return binds, nil + return binds, nil } func (container *Container) createVolumes() error { - binds, err := container.getBindMap() - if err != nil { - return err - } + binds, err := container.getBindMap() + if err != nil { + return err + } volumesDriver := container.runtime.volumes.driver // Create the requested volumes if they don't exist for volPath := range container.Config.Volumes { @@ -824,26 +824,25 @@ func (container *Container) createVolumes() error { } container.Volumes[volPath] = srcPath container.VolumesRW[volPath] = srcRW + // Create the mountpoint - rootVolPath := path.Join(container.RootfsPath(), volPath) - if volIsDir { - if err := os.MkdirAll(rootVolPath, 0755); err != nil { - return err - } + volPath = path.Join(container.RootfsPath(), volPath) + rootVolPath, err := utils.FollowSymlink(volPath, container.RootfsPath()) + if err != nil { + panic(err) } - volPath = path.Join(container.RootfsPath(), volPath) - if _, err := os.Stat(volPath); err != nil { + if _, err := os.Stat(rootVolPath); err != nil { if os.IsNotExist(err) { if volIsDir { - if err := os.MkdirAll(volPath, 0755); err != nil { + if err := os.MkdirAll(rootVolPath, 0755); err != nil { return err } } else { - if err := os.MkdirAll(path.Dir(volPath), 0755); err != nil { + if err := os.MkdirAll(path.Dir(rootVolPath), 0755); err != nil { return err } - if f, err := os.OpenFile(volPath, os.O_CREATE, 0755); err != nil { + if f, err := os.OpenFile(rootVolPath, os.O_CREATE, 0755); err != nil { return err } else { f.Close() diff --git a/utils/fs.go b/utils/fs.go index a7bed8679a..e4897506d9 100644 --- a/utils/fs.go +++ b/utils/fs.go @@ -3,6 +3,7 @@ package utils import ( "os" "path/filepath" + "strings" "syscall" ) @@ -33,3 +34,49 @@ func TreeSize(dir string) (size int64, err error) { }) return } + +// FollowSymlink will follow an existing link and scope it to the root +// path provided. +func FollowSymlinkInScope(link, root string) (string, error) { + prev := "." + + root, err := filepath.Abs(root) + if err != nil { + return "", err + } + root = filepath.Clean(root) + link, err := filepath.Abs(link) + if err != nil { + return "", err + } + link = filepath.Clean(link) + + for _, p := range strings.Split(link, "/") { + prev = filepath.Join(prev, p) + prev = filepath.Clean(prev) + + stat, err := os.Lstat(prev) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", err + } + if stat.Mode()&os.ModeSymlink == os.ModeSymlink { + dest, err := os.Readlink(prev) + if err != nil { + return "", err + } + + switch dest[0] { + case '/': + prev = filepath.Join(root, dest) + case '.': + if prev = filepath.Clean(filepath.Join(filepath.Dir(prev), dest)); len(prev) < len(root) { + prev = filepath.Join(root, filepath.Base(dest)) + } + } + } + } + return prev, nil +} diff --git a/utils/fs_test.go b/utils/fs_test.go new file mode 100644 index 0000000000..5f99ea771d --- /dev/null +++ b/utils/fs_test.go @@ -0,0 +1,84 @@ +package utils + +import ( + "os" + "path/filepath" + "testing" +) + +func abs(p string) string { + o, err := filepath.Abs(p) + if err != nil { + panic(err) + } + return o +} + +func TestFollowSymLinkNormal(t *testing.T) { + link := "testdata/fs/a/d/c/data" + + rewrite, err := FollowSymlink(link, "test") + if err != nil { + t.Fatal(err) + } + + if expected := abs("test/b/c/data"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } +} + +func TestFollowSymLinkRandomString(t *testing.T) { + rewrite, err := FollowSymlink("toto", "test") + if err != nil { + t.Fatal(err) + } + + if rewrite != "toto" { + t.Fatalf("Expected toto got %s", rewrite) + } +} + +func TestFollowSymLinkLastLink(t *testing.T) { + link := "testdata/fs/a/d" + + rewrite, err := FollowSymlink(link, "test") + if err != nil { + t.Fatal(err) + } + + if expected := abs("test/b"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } +} + +func TestFollowSymLinkRelativeLink(t *testing.T) { + link := "testdata/fs/a/e/c/data" + + rewrite, err := FollowSymlink(link, "test") + if err != nil { + t.Fatal(err) + } + + if expected := abs("testdata/fs/a/e/c/data"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } +} + +func TestFollowSymLinkRelativeLinkScope(t *testing.T) { + link := "testdata/fs/a/f" + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + root := filepath.Join(pwd, "testdata") + + rewrite, err := FollowSymlink(link, root) + if err != nil { + t.Fatal(err) + } + + if expected := abs("testdata/test"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } +} diff --git a/utils/testdata/fs/a/d b/utils/testdata/fs/a/d new file mode 120000 index 0000000000..28abc96048 --- /dev/null +++ b/utils/testdata/fs/a/d @@ -0,0 +1 @@ +/b \ No newline at end of file diff --git a/utils/testdata/fs/a/e b/utils/testdata/fs/a/e new file mode 120000 index 0000000000..42532fe13c --- /dev/null +++ b/utils/testdata/fs/a/e @@ -0,0 +1 @@ +../b \ No newline at end of file diff --git a/utils/testdata/fs/a/f b/utils/testdata/fs/a/f new file mode 120000 index 0000000000..21de7edc0a --- /dev/null +++ b/utils/testdata/fs/a/f @@ -0,0 +1 @@ +../../../../test \ No newline at end of file From 8fd9633a6bb136af6666626a76056b1592fef0c7 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 11 Dec 2013 17:12:53 -0800 Subject: [PATCH 077/105] Improve FollowLink to handle recursive link and be more strick --- container.go | 2 +- utils/fs.go | 50 ++++++++++++++++++++++++++----------------- utils/fs_test.go | 39 ++++++++++++++++----------------- utils/testdata/fs/b/h | 1 + utils/testdata/fs/g | 1 + 5 files changed, 52 insertions(+), 41 deletions(-) create mode 120000 utils/testdata/fs/b/h create mode 120000 utils/testdata/fs/g diff --git a/container.go b/container.go index 5959ec7600..e207c3223d 100644 --- a/container.go +++ b/container.go @@ -827,7 +827,7 @@ func (container *Container) createVolumes() error { // Create the mountpoint volPath = path.Join(container.RootfsPath(), volPath) - rootVolPath, err := utils.FollowSymlink(volPath, container.RootfsPath()) + rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.RootfsPath()) if err != nil { panic(err) } diff --git a/utils/fs.go b/utils/fs.go index e4897506d9..e710926210 100644 --- a/utils/fs.go +++ b/utils/fs.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "os" "path/filepath" "strings" @@ -38,43 +39,52 @@ func TreeSize(dir string) (size int64, err error) { // FollowSymlink will follow an existing link and scope it to the root // path provided. func FollowSymlinkInScope(link, root string) (string, error) { - prev := "." + prev := "/" root, err := filepath.Abs(root) if err != nil { return "", err } - root = filepath.Clean(root) - link, err := filepath.Abs(link) + + link, err = filepath.Abs(link) if err != nil { return "", err } - link = filepath.Clean(link) + + if !strings.HasPrefix(filepath.Dir(link), root) { + return "", fmt.Errorf("%s is not within %s", link, root) + } for _, p := range strings.Split(link, "/") { prev = filepath.Join(prev, p) prev = filepath.Clean(prev) - stat, err := os.Lstat(prev) - if err != nil { - if os.IsNotExist(err) { - continue - } - return "", err - } - if stat.Mode()&os.ModeSymlink == os.ModeSymlink { - dest, err := os.Readlink(prev) + for { + stat, err := os.Lstat(prev) if err != nil { + if os.IsNotExist(err) { + break + } return "", err } - - switch dest[0] { - case '/': - prev = filepath.Join(root, dest) - case '.': - if prev = filepath.Clean(filepath.Join(filepath.Dir(prev), dest)); len(prev) < len(root) { - prev = filepath.Join(root, filepath.Base(dest)) + if stat.Mode()&os.ModeSymlink == os.ModeSymlink { + dest, err := os.Readlink(prev) + if err != nil { + return "", err } + + switch dest[0] { + case '/': + prev = filepath.Join(root, dest) + case '.': + prev, _ = filepath.Abs(prev) + + if prev = filepath.Clean(filepath.Join(filepath.Dir(prev), dest)); len(prev) < len(root) { + prev = filepath.Join(root, filepath.Base(dest)) + } + } + } else { + break } } } diff --git a/utils/fs_test.go b/utils/fs_test.go index 5f99ea771d..dd5d97be40 100644 --- a/utils/fs_test.go +++ b/utils/fs_test.go @@ -1,15 +1,14 @@ package utils import ( - "os" "path/filepath" "testing" ) -func abs(p string) string { +func abs(t *testing.T, p string) string { o, err := filepath.Abs(p) if err != nil { - panic(err) + t.Fatal(err) } return o } @@ -17,36 +16,31 @@ func abs(p string) string { func TestFollowSymLinkNormal(t *testing.T) { link := "testdata/fs/a/d/c/data" - rewrite, err := FollowSymlink(link, "test") + rewrite, err := FollowSymlinkInScope(link, "testdata") if err != nil { t.Fatal(err) } - if expected := abs("test/b/c/data"); expected != rewrite { + if expected := abs(t, "testdata/b/c/data"); expected != rewrite { t.Fatalf("Expected %s got %s", expected, rewrite) } } func TestFollowSymLinkRandomString(t *testing.T) { - rewrite, err := FollowSymlink("toto", "test") - if err != nil { - t.Fatal(err) - } - - if rewrite != "toto" { - t.Fatalf("Expected toto got %s", rewrite) + if _, err := FollowSymlinkInScope("toto", "testdata"); err == nil { + t.Fatal("Random string should fail but didn't") } } func TestFollowSymLinkLastLink(t *testing.T) { link := "testdata/fs/a/d" - rewrite, err := FollowSymlink(link, "test") + rewrite, err := FollowSymlinkInScope(link, "testdata") if err != nil { t.Fatal(err) } - if expected := abs("test/b"); expected != rewrite { + if expected := abs(t, "testdata/b"); expected != rewrite { t.Fatalf("Expected %s got %s", expected, rewrite) } } @@ -54,31 +48,36 @@ func TestFollowSymLinkLastLink(t *testing.T) { func TestFollowSymLinkRelativeLink(t *testing.T) { link := "testdata/fs/a/e/c/data" - rewrite, err := FollowSymlink(link, "test") + rewrite, err := FollowSymlinkInScope(link, "testdata") if err != nil { t.Fatal(err) } - if expected := abs("testdata/fs/a/e/c/data"); expected != rewrite { + if expected := abs(t, "testdata/fs/b/c/data"); expected != rewrite { t.Fatalf("Expected %s got %s", expected, rewrite) } } func TestFollowSymLinkRelativeLinkScope(t *testing.T) { link := "testdata/fs/a/f" - pwd, err := os.Getwd() + + rewrite, err := FollowSymlinkInScope(link, "testdata") if err != nil { t.Fatal(err) } - root := filepath.Join(pwd, "testdata") + if expected := abs(t, "testdata/test"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } - rewrite, err := FollowSymlink(link, root) + link = "testdata/fs/b/h" + + rewrite, err = FollowSymlinkInScope(link, "testdata") if err != nil { t.Fatal(err) } - if expected := abs("testdata/test"); expected != rewrite { + if expected := abs(t, "testdata/root"); expected != rewrite { t.Fatalf("Expected %s got %s", expected, rewrite) } } diff --git a/utils/testdata/fs/b/h b/utils/testdata/fs/b/h new file mode 120000 index 0000000000..24387a68fb --- /dev/null +++ b/utils/testdata/fs/b/h @@ -0,0 +1 @@ +../g \ No newline at end of file diff --git a/utils/testdata/fs/g b/utils/testdata/fs/g new file mode 120000 index 0000000000..0ce5de0647 --- /dev/null +++ b/utils/testdata/fs/g @@ -0,0 +1 @@ +../../../../../../../../../../../../root \ No newline at end of file From bf5b949ffc005127399db51f8f49c3c8cc4736bc Mon Sep 17 00:00:00 2001 From: Anton Nikitin Date: Thu, 12 Dec 2013 01:09:24 -0500 Subject: [PATCH 078/105] Minor spelling fix --- docs/sources/use/working_with_volumes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/use/working_with_volumes.rst b/docs/sources/use/working_with_volumes.rst index b5ef991138..341bfe8d6a 100644 --- a/docs/sources/use/working_with_volumes.rst +++ b/docs/sources/use/working_with_volumes.rst @@ -46,7 +46,7 @@ volumes to any container created from the image:: Mount Volumes from an Existing Container: ----------------------------------------- -The command below creates a new container which is runnning as daemon +The command below creates a new container which is running as daemon ``-d`` and with one volume ``/var/lib/couchdb``:: COUCH1=$(sudo docker run -d -v /var/lib/couchdb shykes/couchdb:2013-05-03) From f26a9d456ca267b0346077b98fc447b99319d8b9 Mon Sep 17 00:00:00 2001 From: Zain Memon Date: Thu, 12 Dec 2013 01:23:16 -0800 Subject: [PATCH 079/105] Small typo fixes --- docs/sources/contributing/devenvironment.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index bba43527f7..cc254bbc68 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -135,7 +135,7 @@ You can run an interactive session in the newly built container: # type 'exit' or Ctrl-D to exit -Extra Step: Build and view the Documenation +Extra Step: Build and view the Documentation ------------------------------------------- If you want to read the documentation from a local website, or are making changes @@ -150,4 +150,4 @@ to it, you can build the documentation and then serve it by: **Need More Help?** -If you need more help then hop on to the `#docker-dev IRC channel `_ or post a message on the `Docker developer mailinglist `_. +If you need more help then hop on to the `#docker-dev IRC channel `_ or post a message on the `Docker developer mailing list `_. From f216448c823d1aa3dd661935d60da390d463039d Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 12 Dec 2013 13:03:33 +0200 Subject: [PATCH 080/105] vagrant: update & verify virtualbox guest tools --- Vagrantfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 6bbea51d46..2d6b488cfe 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -79,9 +79,10 @@ if [ ! -d /opt/VBoxGuestAdditions-4.3.2/ ]; then apt-get install -q -y linux-headers-generic-lts-raring dkms echo 'Downloading VBox Guest Additions...' - wget -cq http://dlc.sun.com.edgesuite.net/virtualbox/4.3.2/VBoxGuestAdditions_4.3.2.iso + wget -cq http://dlc.sun.com.edgesuite.net/virtualbox/4.3.4/VBoxGuestAdditions_4.3.4.iso + echo "f120793fa35050a8280eacf9c930cf8d9b88795161520f6515c0cc5edda2fe8a VBoxGuestAdditions_4.3.4.iso" | sha256sum --check || exit 1 - mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.3.2.iso /mnt + mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.3.4.iso /mnt /mnt/VBoxLinuxAdditions.run --nox11 umount /mnt fi From 0db1c60542d3570558bad09eddd3b2b605e55e4e Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 12 Dec 2013 11:24:00 -0700 Subject: [PATCH 081/105] Make Tianon the hack maintainer --- hack/MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/MAINTAINERS b/hack/MAINTAINERS index afd13f8e50..18e05a3070 100644 --- a/hack/MAINTAINERS +++ b/hack/MAINTAINERS @@ -1,2 +1 @@ -Solomon Hykes (@shykes) Tianon Gravi (@tianon) From bd02d6e662fdf197665a92663854c629c4dda747 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 2 Dec 2013 12:58:45 -0600 Subject: [PATCH 082/105] dockerinit: put args in a struct --- sysinit/sysinit.go | 88 ++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 63e91a02e4..19e40bf222 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -16,15 +16,23 @@ import ( "syscall" ) +type DockerInitArgs struct { + user string + gateway string + workDir string + env []string + args []string +} + // Setup networking -func setupNetworking(gw string) { - if gw == "" { +func setupNetworking(args *DockerInitArgs) { + if args.gateway == "" { return } - ip := net.ParseIP(gw) + ip := net.ParseIP(args.gateway) if ip == nil { - log.Fatalf("Unable to set up networking, %s is not a valid IP", gw) + log.Fatalf("Unable to set up networking, %s is not a valid IP", args.gateway) return } @@ -34,23 +42,23 @@ func setupNetworking(gw string) { } // Setup working directory -func setupWorkingDirectory(workdir string) { - if workdir == "" { +func setupWorkingDirectory(args *DockerInitArgs) { + if args.workDir == "" { return } - if err := syscall.Chdir(workdir); err != nil { - log.Fatalf("Unable to change dir to %v: %v", workdir, err) + if err := syscall.Chdir(args.workDir); err != nil { + log.Fatalf("Unable to change dir to %v: %v", args.workDir, err) } } // Takes care of dropping privileges to the desired user -func changeUser(u string) { - if u == "" { +func changeUser(args *DockerInitArgs) { + if args.user == "" { return } - userent, err := utils.UserLookup(u) + userent, err := utils.UserLookup(args.user) if err != nil { - log.Fatalf("Unable to find user %v: %v", u, err) + log.Fatalf("Unable to find user %v: %v", args.user, err) } uid, err := strconv.Atoi(userent.Uid) @@ -71,18 +79,9 @@ func changeUser(u string) { } // Clear environment pollution introduced by lxc-start -func cleanupEnv() { +func setupEnv(args *DockerInitArgs) { os.Clearenv() - var lines []string - content, err := ioutil.ReadFile("/.dockerenv") - if err != nil { - log.Fatalf("Unable to load environment variables: %v", err) - } - err = json.Unmarshal(content, &lines) - if err != nil { - log.Fatalf("Unable to unmarshal environment variables: %v", err) - } - for _, kv := range lines { + for _, kv := range args.env { parts := strings.SplitN(kv, "=", 2) if len(parts) == 1 { parts = append(parts, "") @@ -91,14 +90,14 @@ func cleanupEnv() { } } -func executeProgram(name string, args []string) { - path, err := exec.LookPath(name) +func executeProgram(args *DockerInitArgs) { + path, err := exec.LookPath(args.args[0]) if err != nil { - log.Printf("Unable to locate %v", name) + log.Printf("Unable to locate %v", args.args[0]) os.Exit(127) } - if err := syscall.Exec(path, args, os.Environ()); err != nil { + if err := syscall.Exec(path, args.args, os.Environ()); err != nil { panic(err) } } @@ -111,15 +110,34 @@ func SysInit() { fmt.Println("You should not invoke dockerinit manually") os.Exit(1) } - var u = flag.String("u", "", "username or uid") - var gw = flag.String("g", "", "gateway address") - var workdir = flag.String("w", "", "workdir") + // Get cmdline arguments + user := flag.String("u", "", "username or uid") + gateway := flag.String("g", "", "gateway address") + workDir := flag.String("w", "", "workdir") flag.Parse() - cleanupEnv() - setupNetworking(*gw) - setupWorkingDirectory(*workdir) - changeUser(*u) - executeProgram(flag.Arg(0), flag.Args()) + // Get env + var env []string + content, err := ioutil.ReadFile("/.dockerenv") + if err != nil { + log.Fatalf("Unable to load environment variables: %v", err) + } + if err := json.Unmarshal(content, &env); err != nil { + log.Fatalf("Unable to unmarshal environment variables: %v", err) + } + + args := &DockerInitArgs{ + user: *user, + gateway: *gateway, + workDir: *workDir, + env: env, + args: flag.Args(), + } + + setupEnv(args) + setupNetworking(args) + setupWorkingDirectory(args) + changeUser(args) + executeProgram(args) } From 1572989201cc54dc87d961fe9c5d6850c61d47b9 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 2 Dec 2013 16:21:29 -0600 Subject: [PATCH 083/105] dockerinit: refactor error handling --- sysinit/sysinit.go | 61 ++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 19e40bf222..3f26751ed2 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -25,57 +25,61 @@ type DockerInitArgs struct { } // Setup networking -func setupNetworking(args *DockerInitArgs) { +func setupNetworking(args *DockerInitArgs) error { if args.gateway == "" { - return + return nil } ip := net.ParseIP(args.gateway) if ip == nil { - log.Fatalf("Unable to set up networking, %s is not a valid IP", args.gateway) - return + return fmt.Errorf("Unable to set up networking, %s is not a valid IP", args.gateway) } if err := netlink.AddDefaultGw(ip); err != nil { - log.Fatalf("Unable to set up networking: %v", err) + return fmt.Errorf("Unable to set up networking: %v", err) } + + return nil } // Setup working directory -func setupWorkingDirectory(args *DockerInitArgs) { +func setupWorkingDirectory(args *DockerInitArgs) error { if args.workDir == "" { - return + return nil } if err := syscall.Chdir(args.workDir); err != nil { - log.Fatalf("Unable to change dir to %v: %v", args.workDir, err) + return fmt.Errorf("Unable to change dir to %v: %v", args.workDir, err) } + return nil } // Takes care of dropping privileges to the desired user -func changeUser(args *DockerInitArgs) { +func changeUser(args *DockerInitArgs) error { if args.user == "" { - return + return nil } userent, err := utils.UserLookup(args.user) if err != nil { - log.Fatalf("Unable to find user %v: %v", args.user, err) + return fmt.Errorf("Unable to find user %v: %v", args.user, err) } uid, err := strconv.Atoi(userent.Uid) if err != nil { - log.Fatalf("Invalid uid: %v", userent.Uid) + return fmt.Errorf("Invalid uid: %v", userent.Uid) } gid, err := strconv.Atoi(userent.Gid) if err != nil { - log.Fatalf("Invalid gid: %v", userent.Gid) + return fmt.Errorf("Invalid gid: %v", userent.Gid) } if err := syscall.Setgid(gid); err != nil { - log.Fatalf("setgid failed: %v", err) + return fmt.Errorf("setgid failed: %v", err) } if err := syscall.Setuid(uid); err != nil { - log.Fatalf("setuid failed: %v", err) + return fmt.Errorf("setuid failed: %v", err) } + + return nil } // Clear environment pollution introduced by lxc-start @@ -90,7 +94,21 @@ func setupEnv(args *DockerInitArgs) { } } -func executeProgram(args *DockerInitArgs) { +func executeProgram(args *DockerInitArgs) error { + setupEnv(args) + + if err := setupNetworking(args); err != nil { + return err + } + + if err := setupWorkingDirectory(args); err != nil { + return err + } + + if err := changeUser(args); err != nil { + return err + } + path, err := exec.LookPath(args.args[0]) if err != nil { log.Printf("Unable to locate %v", args.args[0]) @@ -100,6 +118,9 @@ func executeProgram(args *DockerInitArgs) { if err := syscall.Exec(path, args.args, os.Environ()); err != nil { panic(err) } + + // Will never reach here + return nil } // Sys Init code @@ -135,9 +156,7 @@ func SysInit() { args: flag.Args(), } - setupEnv(args) - setupNetworking(args) - setupWorkingDirectory(args) - changeUser(args) - executeProgram(args) + if err := executeProgram(args); err != nil { + log.Fatal(err) + } } From b8f1c7370516a4eec0d3168d594a7d7d92e7f8e9 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 30 Oct 2013 22:24:47 -0500 Subject: [PATCH 084/105] dockerinit: drop capabilities Drop capabilities in dockerinit instead of with lxc utils, since libvirt-lxc doesn't support it. This will also be needed for machine container mode, since dockerinit needs CAP_SYS_ADMIN to setup /dev/console correctly. --- container.go | 4 + hack/vendor.sh | 2 + lxc_template.go | 7 - sysinit/sysinit.go | 64 +- .../github.com/syndtr/gocapability/LICENSE | 24 + .../gocapability/capability/capability.go | 71 +++ .../capability/capability_linux.go | 561 ++++++++++++++++++ .../capability/capability_noop.go | 19 + .../capability/capability_test.go | 83 +++ .../syndtr/gocapability/capability/enum.go | 338 +++++++++++ .../gocapability/capability/syscall_linux.go | 143 +++++ 11 files changed, 1299 insertions(+), 17 deletions(-) create mode 100644 vendor/src/github.com/syndtr/gocapability/LICENSE create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/capability.go create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/capability_linux.go create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/capability_noop.go create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/capability_test.go create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/enum.go create mode 100644 vendor/src/github.com/syndtr/gocapability/capability/syscall_linux.go diff --git a/container.go b/container.go index e207c3223d..2307948e26 100644 --- a/container.go +++ b/container.go @@ -594,6 +594,10 @@ func (container *Container) Start() (err error) { env = append(env, "TERM=xterm") } + if container.hostConfig.Privileged { + params = append(params, "-privileged") + } + // Init any links between the parent and children runtime := container.runtime diff --git a/hack/vendor.sh b/hack/vendor.sh index 93dd8739ce..fc0180c538 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -27,6 +27,8 @@ git_clone github.com/gorilla/context/ 708054d61e5 git_clone github.com/gorilla/mux/ 9b36453141c +git_clone github.com/syndtr/gocapability 3454319be2 + # Docker requires code.google.com/p/go.net/websocket PKG=code.google.com/p/go.net REV=84a4013f96e0 ( diff --git a/lxc_template.go b/lxc_template.go index 2d95a2971d..8e6c6ef1ea 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -110,18 +110,11 @@ lxc.mount.entry = {{escapeFstabSpaces $realPath}} {{escapeFstabSpaces $ROOTFS}}/ {{end}} {{if (getHostConfig .).Privileged}} -# retain all capabilities; no lxc.cap.drop line {{if (getCapabilities .).AppArmor}} lxc.aa_profile = unconfined {{else}} #lxc.aa_profile = unconfined {{end}} -{{else}} -# drop linux capabilities (apply mainly to the user root in the container) -# (Note: 'lxc.cap.keep' is coming soon and should replace this under the -# security principle 'deny all unless explicitly permitted', see -# http://sourceforge.net/mailarchive/message.php?msg_id=31054627 ) -lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod setpcap sys_admin sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config {{end}} # limits diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 3f26751ed2..220f446d62 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/netlink" "github.com/dotcloud/docker/utils" + "github.com/syndtr/gocapability/capability" "io/ioutil" "log" "net" @@ -17,11 +18,12 @@ import ( ) type DockerInitArgs struct { - user string - gateway string - workDir string - env []string - args []string + user string + gateway string + workDir string + privileged bool + env []string + args []string } // Setup networking @@ -82,6 +84,42 @@ func changeUser(args *DockerInitArgs) error { return nil } +func setupCapabilities(args *DockerInitArgs) error { + + if args.privileged { + return nil + } + + drop := []capability.Cap{ + capability.CAP_SETPCAP, + capability.CAP_SYS_MODULE, + capability.CAP_SYS_RAWIO, + capability.CAP_SYS_PACCT, + capability.CAP_SYS_ADMIN, + capability.CAP_SYS_NICE, + capability.CAP_SYS_RESOURCE, + capability.CAP_SYS_TIME, + capability.CAP_SYS_TTY_CONFIG, + capability.CAP_MKNOD, + capability.CAP_AUDIT_WRITE, + capability.CAP_AUDIT_CONTROL, + capability.CAP_MAC_OVERRIDE, + capability.CAP_MAC_ADMIN, + } + + c, err := capability.NewPid(os.Getpid()) + if err != nil { + return err + } + + c.Unset(capability.CAPS|capability.BOUNDS, drop...) + + if err := c.Apply(capability.CAPS | capability.BOUNDS); err != nil { + return err + } + return nil +} + // Clear environment pollution introduced by lxc-start func setupEnv(args *DockerInitArgs) { os.Clearenv() @@ -101,6 +139,10 @@ func executeProgram(args *DockerInitArgs) error { return err } + if err := setupCapabilities(args); err != nil { + return err + } + if err := setupWorkingDirectory(args); err != nil { return err } @@ -136,6 +178,7 @@ func SysInit() { user := flag.String("u", "", "username or uid") gateway := flag.String("g", "", "gateway address") workDir := flag.String("w", "", "workdir") + privileged := flag.Bool("privileged", false, "privileged mode") flag.Parse() // Get env @@ -149,11 +192,12 @@ func SysInit() { } args := &DockerInitArgs{ - user: *user, - gateway: *gateway, - workDir: *workDir, - env: env, - args: flag.Args(), + user: *user, + gateway: *gateway, + workDir: *workDir, + privileged: *privileged, + env: env, + args: flag.Args(), } if err := executeProgram(args); err != nil { diff --git a/vendor/src/github.com/syndtr/gocapability/LICENSE b/vendor/src/github.com/syndtr/gocapability/LICENSE new file mode 100644 index 0000000000..80dd96de77 --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/LICENSE @@ -0,0 +1,24 @@ +Copyright 2013 Suryandaru Triandana +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/src/github.com/syndtr/gocapability/capability/capability.go b/vendor/src/github.com/syndtr/gocapability/capability/capability.go new file mode 100644 index 0000000000..9df3b4151b --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/capability.go @@ -0,0 +1,71 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Package capability provides utilities for manipulating POSIX capabilities. +package capability + +type Capabilities interface { + // Get check whether a capability present in the given + // capabilities set. The 'which' value should be one of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + Get(which CapType, what Cap) bool + + // Empty check whether all capability bits of the given capabilities + // set are zero. The 'which' value should be one of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + Empty(which CapType) bool + + // Full check whether all capability bits of the given capabilities + // set are one. The 'which' value should be one of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + Full(which CapType) bool + + // Set sets capabilities of the given capabilities sets. The + // 'which' value should be one or combination (OR'ed) of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + Set(which CapType, caps ...Cap) + + // Unset unsets capabilities of the given capabilities sets. The + // 'which' value should be one or combination (OR'ed) of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + Unset(which CapType, caps ...Cap) + + // Fill sets all bits of the given capabilities kind to one. The + // 'kind' value should be one or combination (OR'ed) of CAPS or + // BOUNDS. + Fill(kind CapType) + + // Clear sets all bits of the given capabilities kind to zero. The + // 'kind' value should be one or combination (OR'ed) of CAPS or + // BOUNDS. + Clear(kind CapType) + + // String return current capabilities state of the given capabilities + // set as string. The 'which' value should be one of EFFECTIVE, + // PERMITTED, INHERITABLE or BOUNDING. + StringCap(which CapType) string + + // String return current capabilities state as string. + String() string + + // Load load actual capabilities value. This will overwrite all + // outstanding changes. + Load() error + + // Apply apply the capabilities settings, so all changes will take + // effect. + Apply(kind CapType) error +} + +// NewPid create new initialized Capabilities object for given pid. +func NewPid(pid int) (Capabilities, error) { + return newPid(pid) +} + +// NewFile create new initialized Capabilities object for given named file. +func NewFile(name string) (Capabilities, error) { + return newFile(name) +} diff --git a/vendor/src/github.com/syndtr/gocapability/capability/capability_linux.go b/vendor/src/github.com/syndtr/gocapability/capability/capability_linux.go new file mode 100644 index 0000000000..3aaae5973a --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/capability_linux.go @@ -0,0 +1,561 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package capability + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strings" + "syscall" +) + +var errUnknownVers = errors.New("unknown capability version") + +const ( + linuxCapVer1 = 0x19980330 + linuxCapVer2 = 0x20071026 + linuxCapVer3 = 0x20080522 +) + +var capVers uint32 + +func init() { + var hdr capHeader + capget(&hdr, nil) + capVers = hdr.version +} + +func mkStringCap(c Capabilities, which CapType) (ret string) { + for i, first := Cap(0), true; i <= CAP_LAST_CAP; i++ { + if !c.Get(which, i) { + continue + } + if first { + first = false + } else { + ret += ", " + } + ret += i.String() + } + return +} + +func mkString(c Capabilities, max CapType) (ret string) { + ret = "{" + for i := CapType(1); i <= max; i <<= 1 { + ret += " " + i.String() + "=\"" + if c.Empty(i) { + ret += "empty" + } else if c.Full(i) { + ret += "full" + } else { + ret += c.StringCap(i) + } + ret += "\"" + } + ret += " }" + return +} + +func newPid(pid int) (c Capabilities, err error) { + switch capVers { + case linuxCapVer1: + p := new(capsV1) + p.hdr.version = capVers + p.hdr.pid = pid + c = p + case linuxCapVer2, linuxCapVer3: + p := new(capsV3) + p.hdr.version = capVers + p.hdr.pid = pid + c = p + default: + err = errUnknownVers + return + } + err = c.Load() + if err != nil { + c = nil + } + return +} + +type capsV1 struct { + hdr capHeader + data capData +} + +func (c *capsV1) Get(which CapType, what Cap) bool { + if what > 32 { + return false + } + + switch which { + case EFFECTIVE: + return (1< 32 { + continue + } + + if which&EFFECTIVE != 0 { + c.data.effective |= 1 << uint(what) + } + if which&PERMITTED != 0 { + c.data.permitted |= 1 << uint(what) + } + if which&INHERITABLE != 0 { + c.data.inheritable |= 1 << uint(what) + } + } +} + +func (c *capsV1) Unset(which CapType, caps ...Cap) { + for _, what := range caps { + if what > 32 { + continue + } + + if which&EFFECTIVE != 0 { + c.data.effective &= ^(1 << uint(what)) + } + if which&PERMITTED != 0 { + c.data.permitted &= ^(1 << uint(what)) + } + if which&INHERITABLE != 0 { + c.data.inheritable &= ^(1 << uint(what)) + } + } +} + +func (c *capsV1) Fill(kind CapType) { + if kind&CAPS == CAPS { + c.data.effective = 0x7fffffff + c.data.permitted = 0x7fffffff + c.data.inheritable = 0 + } +} + +func (c *capsV1) Clear(kind CapType) { + if kind&CAPS == CAPS { + c.data.effective = 0 + c.data.permitted = 0 + c.data.inheritable = 0 + } +} + +func (c *capsV1) StringCap(which CapType) (ret string) { + return mkStringCap(c, which) +} + +func (c *capsV1) String() (ret string) { + return mkString(c, BOUNDING) +} + +func (c *capsV1) Load() (err error) { + return capget(&c.hdr, &c.data) +} + +func (c *capsV1) Apply(kind CapType) error { + if kind&CAPS == CAPS { + return capset(&c.hdr, &c.data) + } + return nil +} + +type capsV3 struct { + hdr capHeader + data [2]capData + bounds [2]uint32 +} + +func (c *capsV3) Get(which CapType, what Cap) bool { + var i uint + if what > 31 { + i = uint(what) >> 5 + what %= 32 + } + + switch which { + case EFFECTIVE: + return (1< 31 { + i = uint(what) >> 5 + what %= 32 + } + + if which&EFFECTIVE != 0 { + c.data[i].effective |= 1 << uint(what) + } + if which&PERMITTED != 0 { + c.data[i].permitted |= 1 << uint(what) + } + if which&INHERITABLE != 0 { + c.data[i].inheritable |= 1 << uint(what) + } + if which&BOUNDING != 0 { + c.bounds[i] |= 1 << uint(what) + } + } +} + +func (c *capsV3) Unset(which CapType, caps ...Cap) { + for _, what := range caps { + var i uint + if what > 31 { + i = uint(what) >> 5 + what %= 32 + } + + if which&EFFECTIVE != 0 { + c.data[i].effective &= ^(1 << uint(what)) + } + if which&PERMITTED != 0 { + c.data[i].permitted &= ^(1 << uint(what)) + } + if which&INHERITABLE != 0 { + c.data[i].inheritable &= ^(1 << uint(what)) + } + if which&BOUNDING != 0 { + c.bounds[i] &= ^(1 << uint(what)) + } + } +} + +func (c *capsV3) Fill(kind CapType) { + if kind&CAPS == CAPS { + c.data[0].effective = 0xffffffff + c.data[0].permitted = 0xffffffff + c.data[0].inheritable = 0 + c.data[1].effective = 0xffffffff + c.data[1].permitted = 0xffffffff + c.data[1].inheritable = 0 + } + + if kind&BOUNDS == BOUNDS { + c.bounds[0] = 0xffffffff + c.bounds[1] = 0xffffffff + } +} + +func (c *capsV3) Clear(kind CapType) { + if kind&CAPS == CAPS { + c.data[0].effective = 0 + c.data[0].permitted = 0 + c.data[0].inheritable = 0 + c.data[1].effective = 0 + c.data[1].permitted = 0 + c.data[1].inheritable = 0 + } + + if kind&BOUNDS == BOUNDS { + c.bounds[0] = 0 + c.bounds[1] = 0 + } +} + +func (c *capsV3) StringCap(which CapType) (ret string) { + return mkStringCap(c, which) +} + +func (c *capsV3) String() (ret string) { + return mkString(c, BOUNDING) +} + +func (c *capsV3) Load() (err error) { + err = capget(&c.hdr, &c.data[0]) + if err != nil { + return + } + + f, err := os.Open(fmt.Sprintf("/proc/%d/status", c.hdr.pid)) + if err != nil { + return + } + b := bufio.NewReader(f) + for { + line, e := b.ReadString('\n') + if e != nil { + if e != io.EOF { + err = e + } + break + } + if strings.HasPrefix(line, "CapB") { + fmt.Sscanf(line[4:], "nd: %08x%08x", &c.bounds[1], &c.bounds[0]) + break + } + } + f.Close() + + return +} + +func (c *capsV3) Apply(kind CapType) (err error) { + if kind&BOUNDS == BOUNDS { + var data [2]capData + err = capget(&c.hdr, &data[0]) + if err != nil { + return + } + if (1< 31 { + if c.data.version == 1 { + return false + } + i = uint(what) >> 5 + what %= 32 + } + + switch which { + case EFFECTIVE: + return (1< 31 { + if c.data.version == 1 { + continue + } + i = uint(what) >> 5 + what %= 32 + } + + if which&EFFECTIVE != 0 { + c.data.effective[i] |= 1 << uint(what) + } + if which&PERMITTED != 0 { + c.data.data[i].permitted |= 1 << uint(what) + } + if which&INHERITABLE != 0 { + c.data.data[i].inheritable |= 1 << uint(what) + } + } +} + +func (c *capsFile) Unset(which CapType, caps ...Cap) { + for _, what := range caps { + var i uint + if what > 31 { + if c.data.version == 1 { + continue + } + i = uint(what) >> 5 + what %= 32 + } + + if which&EFFECTIVE != 0 { + c.data.effective[i] &= ^(1 << uint(what)) + } + if which&PERMITTED != 0 { + c.data.data[i].permitted &= ^(1 << uint(what)) + } + if which&INHERITABLE != 0 { + c.data.data[i].inheritable &= ^(1 << uint(what)) + } + } +} + +func (c *capsFile) Fill(kind CapType) { + if kind&CAPS == CAPS { + c.data.effective[0] = 0xffffffff + c.data.data[0].permitted = 0xffffffff + c.data.data[0].inheritable = 0 + if c.data.version == 2 { + c.data.effective[1] = 0xffffffff + c.data.data[1].permitted = 0xffffffff + c.data.data[1].inheritable = 0 + } + } +} + +func (c *capsFile) Clear(kind CapType) { + if kind&CAPS == CAPS { + c.data.effective[0] = 0 + c.data.data[0].permitted = 0 + c.data.data[0].inheritable = 0 + if c.data.version == 2 { + c.data.effective[1] = 0 + c.data.data[1].permitted = 0 + c.data.data[1].inheritable = 0 + } + } +} + +func (c *capsFile) StringCap(which CapType) (ret string) { + return mkStringCap(c, which) +} + +func (c *capsFile) String() (ret string) { + return mkString(c, INHERITABLE) +} + +func (c *capsFile) Load() (err error) { + return getVfsCap(c.path, &c.data) +} + +func (c *capsFile) Apply(kind CapType) (err error) { + if kind&CAPS == CAPS { + return setVfsCap(c.path, &c.data) + } + return +} diff --git a/vendor/src/github.com/syndtr/gocapability/capability/capability_noop.go b/vendor/src/github.com/syndtr/gocapability/capability/capability_noop.go new file mode 100644 index 0000000000..9bb3070c5e --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/capability_noop.go @@ -0,0 +1,19 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !linux + +package capability + +import "errors" + +func newPid(pid int) (Capabilities, error) { + return nil, errors.New("not supported") +} + +func newFile(path string) (Capabilities, error) { + return nil, errors.New("not supported") +} diff --git a/vendor/src/github.com/syndtr/gocapability/capability/capability_test.go b/vendor/src/github.com/syndtr/gocapability/capability/capability_test.go new file mode 100644 index 0000000000..8108655c05 --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/capability_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package capability + +import "testing" + +func TestState(t *testing.T) { + testEmpty := func(name string, c Capabilities, whats CapType) { + for i := CapType(1); i <= BOUNDING; i <<= 1 { + if (i&whats) != 0 && !c.Empty(i) { + t.Errorf(name+": capabilities set %q wasn't empty", i) + } + } + } + testFull := func(name string, c Capabilities, whats CapType) { + for i := CapType(1); i <= BOUNDING; i <<= 1 { + if (i&whats) != 0 && !c.Full(i) { + t.Errorf(name+": capabilities set %q wasn't full", i) + } + } + } + testPartial := func(name string, c Capabilities, whats CapType) { + for i := CapType(1); i <= BOUNDING; i <<= 1 { + if (i&whats) != 0 && (c.Empty(i) || c.Full(i)) { + t.Errorf(name+": capabilities set %q wasn't partial", i) + } + } + } + testGet := func(name string, c Capabilities, whats CapType, max Cap) { + for i := CapType(1); i <= BOUNDING; i <<= 1 { + if (i & whats) == 0 { + continue + } + for j := Cap(0); j <= max; j++ { + if !c.Get(i, j) { + t.Errorf(name+": capability %q wasn't found on %q", j, i) + } + } + } + } + + capf := new(capsFile) + capf.data.version = 2 + for _, tc := range []struct { + name string + c Capabilities + sets CapType + max Cap + }{ + {"v1", new(capsV1), EFFECTIVE | PERMITTED, CAP_AUDIT_CONTROL}, + {"v3", new(capsV3), EFFECTIVE | PERMITTED | BOUNDING, CAP_LAST_CAP}, + {"file_v1", new(capsFile), EFFECTIVE | PERMITTED, CAP_AUDIT_CONTROL}, + {"file_v2", capf, EFFECTIVE | PERMITTED, CAP_LAST_CAP}, + } { + testEmpty(tc.name, tc.c, tc.sets) + tc.c.Fill(CAPS | BOUNDS) + testFull(tc.name, tc.c, tc.sets) + testGet(tc.name, tc.c, tc.sets, tc.max) + tc.c.Clear(CAPS | BOUNDS) + testEmpty(tc.name, tc.c, tc.sets) + for i := CapType(1); i <= BOUNDING; i <<= 1 { + for j := Cap(0); j <= CAP_LAST_CAP; j++ { + tc.c.Set(i, j) + } + } + testFull(tc.name, tc.c, tc.sets) + testGet(tc.name, tc.c, tc.sets, tc.max) + for i := CapType(1); i <= BOUNDING; i <<= 1 { + for j := Cap(0); j <= CAP_LAST_CAP; j++ { + tc.c.Unset(i, j) + } + } + testEmpty(tc.name, tc.c, tc.sets) + tc.c.Set(PERMITTED, CAP_CHOWN) + testPartial(tc.name, tc.c, PERMITTED) + tc.c.Clear(CAPS | BOUNDS) + testEmpty(tc.name, tc.c, tc.sets) + } +} diff --git a/vendor/src/github.com/syndtr/gocapability/capability/enum.go b/vendor/src/github.com/syndtr/gocapability/capability/enum.go new file mode 100644 index 0000000000..e2900a4e93 --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/enum.go @@ -0,0 +1,338 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package capability + +type CapType uint + +func (c CapType) String() string { + switch c { + case EFFECTIVE: + return "effective" + case PERMITTED: + return "permitted" + case INHERITABLE: + return "inheritable" + case BOUNDING: + return "bounding" + case CAPS: + return "caps" + } + return "unknown" +} + +const ( + EFFECTIVE CapType = 1 << iota + PERMITTED + INHERITABLE + BOUNDING + + CAPS = EFFECTIVE | PERMITTED | INHERITABLE + BOUNDS = BOUNDING +) + +type Cap int + +func (c Cap) String() string { + switch c { + case CAP_CHOWN: + return "chown" + case CAP_DAC_OVERRIDE: + return "dac_override" + case CAP_DAC_READ_SEARCH: + return "dac_read_search" + case CAP_FOWNER: + return "fowner" + case CAP_FSETID: + return "fsetid" + case CAP_KILL: + return "kill" + case CAP_SETGID: + return "setgid" + case CAP_SETUID: + return "setuid" + case CAP_SETPCAP: + return "setpcap" + case CAP_LINUX_IMMUTABLE: + return "linux_immutable" + case CAP_NET_BIND_SERVICE: + return "net_bind_service" + case CAP_NET_BROADCAST: + return "net_broadcast" + case CAP_NET_ADMIN: + return "net_admin" + case CAP_NET_RAW: + return "net_raw" + case CAP_IPC_LOCK: + return "ipc_lock" + case CAP_IPC_OWNER: + return "ipc_owner" + case CAP_SYS_MODULE: + return "sys_module" + case CAP_SYS_RAWIO: + return "sys_rawio" + case CAP_SYS_CHROOT: + return "sys_chroot" + case CAP_SYS_PTRACE: + return "sys_ptrace" + case CAP_SYS_PACCT: + return "sys_psacct" + case CAP_SYS_ADMIN: + return "sys_admin" + case CAP_SYS_BOOT: + return "sys_boot" + case CAP_SYS_NICE: + return "sys_nice" + case CAP_SYS_RESOURCE: + return "sys_resource" + case CAP_SYS_TIME: + return "sys_time" + case CAP_SYS_TTY_CONFIG: + return "sys_tty_config" + case CAP_MKNOD: + return "mknod" + case CAP_LEASE: + return "lease" + case CAP_AUDIT_WRITE: + return "audit_write" + case CAP_AUDIT_CONTROL: + return "audit_control" + case CAP_SETFCAP: + return "setfcap" + case CAP_MAC_OVERRIDE: + return "mac_override" + case CAP_MAC_ADMIN: + return "mac_admin" + case CAP_SYSLOG: + return "syslog" + case CAP_WAKE_ALARM: + return "wake_alarm" + case CAP_BLOCK_SUSPEND: + return "block_suspend" + } + return "unknown" +} + +const ( + // POSIX-draft defined capabilities. + + // In a system with the [_POSIX_CHOWN_RESTRICTED] option defined, this + // overrides the restriction of changing file ownership and group + // ownership. + CAP_CHOWN Cap = 0 + + // Override all DAC access, including ACL execute access if + // [_POSIX_ACL] is defined. Excluding DAC access covered by + // CAP_LINUX_IMMUTABLE. + CAP_DAC_OVERRIDE Cap = 1 + + // Overrides all DAC restrictions regarding read and search on files + // and directories, including ACL restrictions if [_POSIX_ACL] is + // defined. Excluding DAC access covered by CAP_LINUX_IMMUTABLE. + CAP_DAC_READ_SEARCH Cap = 2 + + // Overrides all restrictions about allowed operations on files, where + // file owner ID must be equal to the user ID, except where CAP_FSETID + // is applicable. It doesn't override MAC and DAC restrictions. + CAP_FOWNER Cap = 3 + + // Overrides the following restrictions that the effective user ID + // shall match the file owner ID when setting the S_ISUID and S_ISGID + // bits on that file; that the effective group ID (or one of the + // supplementary group IDs) shall match the file owner ID when setting + // the S_ISGID bit on that file; that the S_ISUID and S_ISGID bits are + // cleared on successful return from chown(2) (not implemented). + CAP_FSETID Cap = 4 + + // Overrides the restriction that the real or effective user ID of a + // process sending a signal must match the real or effective user ID + // of the process receiving the signal. + CAP_KILL Cap = 5 + + // Allows setgid(2) manipulation + // Allows setgroups(2) + // Allows forged gids on socket credentials passing. + CAP_SETGID Cap = 6 + + // Allows set*uid(2) manipulation (including fsuid). + // Allows forged pids on socket credentials passing. + CAP_SETUID Cap = 7 + + // Linux-specific capabilities + + // Without VFS support for capabilities: + // Transfer any capability in your permitted set to any pid, + // remove any capability in your permitted set from any pid + // With VFS support for capabilities (neither of above, but) + // Add any capability from current's capability bounding set + // to the current process' inheritable set + // Allow taking bits out of capability bounding set + // Allow modification of the securebits for a process + CAP_SETPCAP Cap = 8 + + // Allow modification of S_IMMUTABLE and S_APPEND file attributes + CAP_LINUX_IMMUTABLE Cap = 9 + + // Allows binding to TCP/UDP sockets below 1024 + // Allows binding to ATM VCIs below 32 + CAP_NET_BIND_SERVICE Cap = 10 + + // Allow broadcasting, listen to multicast + CAP_NET_BROADCAST Cap = 11 + + // Allow interface configuration + // Allow administration of IP firewall, masquerading and accounting + // Allow setting debug option on sockets + // Allow modification of routing tables + // Allow setting arbitrary process / process group ownership on + // sockets + // Allow binding to any address for transparent proxying (also via NET_RAW) + // Allow setting TOS (type of service) + // Allow setting promiscuous mode + // Allow clearing driver statistics + // Allow multicasting + // Allow read/write of device-specific registers + // Allow activation of ATM control sockets + CAP_NET_ADMIN Cap = 12 + + // Allow use of RAW sockets + // Allow use of PACKET sockets + // Allow binding to any address for transparent proxying (also via NET_ADMIN) + CAP_NET_RAW Cap = 13 + + // Allow locking of shared memory segments + // Allow mlock and mlockall (which doesn't really have anything to do + // with IPC) + CAP_IPC_LOCK Cap = 14 + + // Override IPC ownership checks + CAP_IPC_OWNER Cap = 15 + + // Insert and remove kernel modules - modify kernel without limit + CAP_SYS_MODULE Cap = 16 + + // Allow ioperm/iopl access + // Allow sending USB messages to any device via /proc/bus/usb + CAP_SYS_RAWIO Cap = 17 + + // Allow use of chroot() + CAP_SYS_CHROOT Cap = 18 + + // Allow ptrace() of any process + CAP_SYS_PTRACE Cap = 19 + + // Allow configuration of process accounting + CAP_SYS_PACCT Cap = 20 + + // Allow configuration of the secure attention key + // Allow administration of the random device + // Allow examination and configuration of disk quotas + // Allow setting the domainname + // Allow setting the hostname + // Allow calling bdflush() + // Allow mount() and umount(), setting up new smb connection + // Allow some autofs root ioctls + // Allow nfsservctl + // Allow VM86_REQUEST_IRQ + // Allow to read/write pci config on alpha + // Allow irix_prctl on mips (setstacksize) + // Allow flushing all cache on m68k (sys_cacheflush) + // Allow removing semaphores + // Used instead of CAP_CHOWN to "chown" IPC message queues, semaphores + // and shared memory + // Allow locking/unlocking of shared memory segment + // Allow turning swap on/off + // Allow forged pids on socket credentials passing + // Allow setting readahead and flushing buffers on block devices + // Allow setting geometry in floppy driver + // Allow turning DMA on/off in xd driver + // Allow administration of md devices (mostly the above, but some + // extra ioctls) + // Allow tuning the ide driver + // Allow access to the nvram device + // Allow administration of apm_bios, serial and bttv (TV) device + // Allow manufacturer commands in isdn CAPI support driver + // Allow reading non-standardized portions of pci configuration space + // Allow DDI debug ioctl on sbpcd driver + // Allow setting up serial ports + // Allow sending raw qic-117 commands + // Allow enabling/disabling tagged queuing on SCSI controllers and sending + // arbitrary SCSI commands + // Allow setting encryption key on loopback filesystem + // Allow setting zone reclaim policy + CAP_SYS_ADMIN Cap = 21 + + // Allow use of reboot() + CAP_SYS_BOOT Cap = 22 + + // Allow raising priority and setting priority on other (different + // UID) processes + // Allow use of FIFO and round-robin (realtime) scheduling on own + // processes and setting the scheduling algorithm used by another + // process. + // Allow setting cpu affinity on other processes + CAP_SYS_NICE Cap = 23 + + // Override resource limits. Set resource limits. + // Override quota limits. + // Override reserved space on ext2 filesystem + // Modify data journaling mode on ext3 filesystem (uses journaling + // resources) + // NOTE: ext2 honors fsuid when checking for resource overrides, so + // you can override using fsuid too + // Override size restrictions on IPC message queues + // Allow more than 64hz interrupts from the real-time clock + // Override max number of consoles on console allocation + // Override max number of keymaps + CAP_SYS_RESOURCE Cap = 24 + + // Allow manipulation of system clock + // Allow irix_stime on mips + // Allow setting the real-time clock + CAP_SYS_TIME Cap = 25 + + // Allow configuration of tty devices + // Allow vhangup() of tty + CAP_SYS_TTY_CONFIG Cap = 26 + + // Allow the privileged aspects of mknod() + CAP_MKNOD Cap = 27 + + // Allow taking of leases on files + CAP_LEASE Cap = 28 + + CAP_AUDIT_WRITE Cap = 29 + CAP_AUDIT_CONTROL Cap = 30 + CAP_SETFCAP Cap = 31 + + // Override MAC access. + // The base kernel enforces no MAC policy. + // An LSM may enforce a MAC policy, and if it does and it chooses + // to implement capability based overrides of that policy, this is + // the capability it should use to do so. + CAP_MAC_OVERRIDE Cap = 32 + + // Allow MAC configuration or state changes. + // The base kernel requires no MAC configuration. + // An LSM may enforce a MAC policy, and if it does and it chooses + // to implement capability based checks on modifications to that + // policy or the data required to maintain it, this is the + // capability it should use to do so. + CAP_MAC_ADMIN Cap = 33 + + // Allow configuring the kernel's syslog (printk behaviour) + CAP_SYSLOG Cap = 34 + + // Allow triggering something that will wake the system + CAP_WAKE_ALARM Cap = 35 + + // Allow preventing system suspends + CAP_BLOCK_SUSPEND Cap = 36 + + CAP_LAST_CAP = CAP_BLOCK_SUSPEND +) + +const capUpperMask = (uint32(1) << (uint(CAP_LAST_CAP) - 31)) - 1 diff --git a/vendor/src/github.com/syndtr/gocapability/capability/syscall_linux.go b/vendor/src/github.com/syndtr/gocapability/capability/syscall_linux.go new file mode 100644 index 0000000000..c18e6f6918 --- /dev/null +++ b/vendor/src/github.com/syndtr/gocapability/capability/syscall_linux.go @@ -0,0 +1,143 @@ +// Copyright (c) 2013, Suryandaru Triandana +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package capability + +import ( + "syscall" + "unsafe" +) + +type capHeader struct { + version uint32 + pid int +} + +type capData struct { + effective uint32 + permitted uint32 + inheritable uint32 +} + +func capget(hdr *capHeader, data *capData) (err error) { + _, _, e1 := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(hdr)), uintptr(unsafe.Pointer(data)), 0) + if e1 != 0 { + err = e1 + } + return +} + +func capset(hdr *capHeader, data *capData) (err error) { + _, _, e1 := syscall.Syscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(hdr)), uintptr(unsafe.Pointer(data)), 0) + if e1 != 0 { + err = e1 + } + return +} + +func prctl(option int, arg2, arg3, arg4, arg5 uintptr) (err error) { + _, _, e1 := syscall.Syscall6(syscall.SYS_PRCTL, uintptr(option), arg2, arg3, arg4, arg5, 0) + if e1 != 0 { + err = e1 + } + return +} + +const ( + vfsXattrName = "security.capability" + + vfsCapVerMask = 0xff000000 + vfsCapVer1 = 0x01000000 + vfsCapVer2 = 0x02000000 + + vfsCapFlagMask = ^vfsCapVerMask + vfsCapFlageffective = 0x000001 + + vfscapDataSizeV1 = 4 * (1 + 2*1) + vfscapDataSizeV2 = 4 * (1 + 2*2) +) + +type vfscapData struct { + magic uint32 + data [2]struct { + permitted uint32 + inheritable uint32 + } + effective [2]uint32 + version int8 +} + +var ( + _vfsXattrName *byte +) + +func init() { + _vfsXattrName, _ = syscall.BytePtrFromString(vfsXattrName) +} + +func getVfsCap(path string, dest *vfscapData) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_vfsXattrName)), uintptr(unsafe.Pointer(dest)), vfscapDataSizeV2, 0, 0) + if e1 != 0 { + err = e1 + } + switch dest.magic & vfsCapVerMask { + case vfsCapVer1: + dest.version = 1 + if r0 != vfscapDataSizeV1 { + return syscall.EINVAL + } + dest.data[1].permitted = 0 + dest.data[1].inheritable = 0 + case vfsCapVer2: + dest.version = 2 + if r0 != vfscapDataSizeV2 { + return syscall.EINVAL + } + default: + return syscall.EINVAL + } + if dest.magic&vfsCapFlageffective != 0 { + dest.effective[0] = dest.data[0].permitted | dest.data[0].inheritable + dest.effective[1] = dest.data[1].permitted | dest.data[1].inheritable + } else { + dest.effective[0] = 0 + dest.effective[1] = 0 + } + return +} + +func setVfsCap(path string, data *vfscapData) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var size uintptr + if data.version == 1 { + data.magic = vfsCapVer1 + size = vfscapDataSizeV1 + } else if data.version == 2 { + data.magic = vfsCapVer2 + if data.effective[0] != 0 || data.effective[1] != 0 { + data.magic |= vfsCapFlageffective + data.data[0].permitted |= data.effective[0] + data.data[1].permitted |= data.effective[1] + } + size = vfscapDataSizeV2 + } else { + return syscall.EINVAL + } + _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_vfsXattrName)), uintptr(unsafe.Pointer(data)), size, 0, 0) + if e1 != 0 { + err = e1 + } + return +} From e43ff2f6f2630f971c9494aed285f1d410e42df9 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 11 Dec 2013 17:52:41 -0800 Subject: [PATCH 085/105] move tag to job --- api.go | 11 +++-------- integration/api_test.go | 2 +- integration/commands_test.go | 3 +-- integration/server_test.go | 19 ++++++++++--------- server.go | 22 +++++++++++++++++----- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index 58fa310240..f7a7ac91a4 100644 --- a/api.go +++ b/api.go @@ -356,18 +356,13 @@ func postImagesTag(srv *Server, version float64, w http.ResponseWriter, r *http. if err := parseForm(r); err != nil { return err } - repo := r.Form.Get("repo") - tag := r.Form.Get("tag") if vars == nil { return fmt.Errorf("Missing parameter") } - name := vars["name"] - force, err := getBoolParam(r.Form.Get("force")) - if err != nil { - return err - } - if err := srv.ContainerTag(name, repo, tag, force); err != nil { + job := srv.Eng.Job("tag", vars["name"], r.Form.Get("repo"), r.Form.Get("tag")) + job.Setenv("force", r.Form.Get("force")) + if err := job.Run(); err != nil { return err } w.WriteHeader(http.StatusCreated) diff --git a/integration/api_test.go b/integration/api_test.go index 819c43a8a5..eed17c9084 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -1120,7 +1120,7 @@ func TestDeleteImages(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "test", "test", false); err != nil { + if err := eng.Job("tag", unitTestImageName, "test", "test").Run(); err != nil { t.Fatal(err) } images, err := srv.Images(false, "") diff --git a/integration/commands_test.go b/integration/commands_test.go index 16e2b3fb02..822567a7fa 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -910,8 +910,7 @@ run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] t.Fatal(err) } - err = mkServerFromEngine(eng, t).ContainerTag(image.ID, "test", "latest", false) - if err != nil { + if err := eng.Job("tag", image.ID, "test").Run(); err != nil { t.Fatal(err) } diff --git a/integration/server_test.go b/integration/server_test.go index c1499e26cc..9cb26a4c2d 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -8,7 +8,7 @@ import ( "testing" ) -func TestContainerTagImageDelete(t *testing.T) { +func TestImageTagImageDelete(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() @@ -18,14 +18,15 @@ func TestContainerTagImageDelete(t *testing.T) { if err != nil { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "utest", "tag1", false); err != nil { + if err := eng.Job("tag", unitTestImageName, "utest", "tag1").Run(); err != nil { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "utest/docker", "tag2", false); err != nil { + if err := eng.Job("tag", unitTestImageName, "utest/docker", "tag2").Run(); err != nil { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "utest:5000/docker", "tag3", false); err != nil { + + if err := eng.Job("tag", unitTestImageName, "utest:5000/docker", "tag3").Run(); err != nil { t.Fatal(err) } @@ -265,8 +266,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - err = srv.ContainerTag(imageID, "test", "0.1", false) - if err != nil { + if err := eng.Job("tag", imageID, "test", "0.1").Run(); err != nil { t.Fatal(err) } @@ -329,14 +329,15 @@ func TestImagesFilter(t *testing.T) { srv := mkServerFromEngine(eng, t) - if err := srv.ContainerTag(unitTestImageName, "utest", "tag1", false); err != nil { + if err := eng.Job("tag", unitTestImageName, "utest", "tag1").Run(); err != nil { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "utest/docker", "tag2", false); err != nil { + if err := eng.Job("tag", unitTestImageName, "utest/docker", "tag2").Run(); err != nil { t.Fatal(err) } - if err := srv.ContainerTag(unitTestImageName, "utest:5000/docker", "tag3", false); err != nil { + + if err := eng.Job("tag", unitTestImageName, "utest:5000/docker", "tag3").Run(); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index c32cfe23ed..bfc4564ac0 100644 --- a/server.go +++ b/server.go @@ -95,6 +95,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("tag", srv.ImageTag); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -769,12 +773,20 @@ func (srv *Server) ContainerCommit(name, repo, tag, author, comment string, conf return img.ID, err } -// FIXME: this should be called ImageTag -func (srv *Server) ContainerTag(name, repo, tag string, force bool) error { - if err := srv.runtime.repositories.Set(repo, tag, name, force); err != nil { - return err +func (srv *Server) ImageTag(job *engine.Job) engine.Status { + if len(job.Args) != 2 && len(job.Args) != 3 { + job.Errorf("Usage: %s IMAGE REPOSITORY [TAG]\n", job.Name) + return engine.StatusErr } - return nil + var tag string + if len(job.Args) == 3 { + tag = job.Args[2] + } + if err := srv.runtime.repositories.Set(job.Args[1], tag, job.Args[0], job.GetenvBool("force")); err != nil { + job.Error(err) + return engine.StatusErr + } + return engine.StatusOK } func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) error { From f7c7f7978cf0ca94da5cc68ec2634b5a22ae329c Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Tue, 29 Oct 2013 13:31:00 -0500 Subject: [PATCH 086/105] dockerinit: set hostname Set the hostname in dockerinit instead of with lxc utils. libvirt-lxc doesn't have a way to do this, so do it in a common place. --- lxc_template.go | 7 ------- lxc_template_unit_test.go | 2 -- sysinit/sysinit.go | 22 ++++++++++++++++++++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lxc_template.go b/lxc_template.go index 8e6c6ef1ea..a8e11967d1 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -6,13 +6,6 @@ import ( ) const LxcTemplate = ` -# hostname -{{if .Config.Hostname}} -lxc.utsname = {{.Config.Hostname}} -{{else}} -lxc.utsname = {{.Id}} -{{end}} - {{if .Config.NetworkDisabled}} # network is disabled (-n=false) lxc.network.type = empty diff --git a/lxc_template_unit_test.go b/lxc_template_unit_test.go index ccdfec8890..f71f1dd6f5 100644 --- a/lxc_template_unit_test.go +++ b/lxc_template_unit_test.go @@ -29,7 +29,6 @@ func TestLXCConfig(t *testing.T) { container := &Container{ root: root, Config: &Config{ - Hostname: "foobar", Memory: int64(mem), CpuShares: int64(cpu), NetworkDisabled: true, @@ -41,7 +40,6 @@ func TestLXCConfig(t *testing.T) { if err := container.generateLXCConfig(); err != nil { t.Fatal(err) } - grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar") grepFile(t, container.lxcConfigPath(), fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) grepFile(t, container.lxcConfigPath(), diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 220f446d62..d6ba3a5d83 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -26,6 +26,14 @@ type DockerInitArgs struct { args []string } +func setupHostname(args *DockerInitArgs) error { + hostname := getEnv(args, "HOSTNAME") + if hostname == "" { + return nil + } + return syscall.Sethostname([]byte(hostname)) +} + // Setup networking func setupNetworking(args *DockerInitArgs) error { if args.gateway == "" { @@ -132,9 +140,23 @@ func setupEnv(args *DockerInitArgs) { } } +func getEnv(args *DockerInitArgs, key string) string { + for _, kv := range args.env { + parts := strings.SplitN(kv, "=", 2) + if parts[0] == key && len(parts) == 2 { + return parts[1] + } + } + return "" +} + func executeProgram(args *DockerInitArgs) error { setupEnv(args) + if err := setupHostname(args); err != nil { + return err + } + if err := setupNetworking(args); err != nil { return err } From ecc51cd465da56b38190fe852cf9f31bdfbb0fdc Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Tue, 29 Oct 2013 13:37:00 -0500 Subject: [PATCH 087/105] dockerinit: set IP address Set the IP address in dockerinit instead of lxc utils, to prepare for using libvirt-lxc. --- container.go | 6 +++++- lxc_template.go | 2 -- sysinit/sysinit.go | 43 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/container.go b/container.go index 2307948e26..9a502dc190 100644 --- a/container.go +++ b/container.go @@ -574,7 +574,11 @@ func (container *Container) Start() (err error) { // Networking if !container.Config.NetworkDisabled { - params = append(params, "-g", container.network.Gateway.String()) + network := container.NetworkSettings + params = append(params, + "-g", network.Gateway, + "-i", fmt.Sprintf("%s/%d", network.IPAddress, network.IPPrefixLen), + ) } // User diff --git a/lxc_template.go b/lxc_template.go index a8e11967d1..ba5a8d5b82 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -12,11 +12,9 @@ lxc.network.type = empty {{else}} # network configuration lxc.network.type = veth -lxc.network.flags = up lxc.network.link = {{.NetworkSettings.Bridge}} lxc.network.name = eth0 lxc.network.mtu = 1500 -lxc.network.ipv4 = {{.NetworkSettings.IPAddress}}/{{.NetworkSettings.IPPrefixLen}} {{end}} # root filesystem diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index d6ba3a5d83..e7a9e229ec 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -20,6 +20,7 @@ import ( type DockerInitArgs struct { user string gateway string + ip string workDir string privileged bool env []string @@ -36,17 +37,41 @@ func setupHostname(args *DockerInitArgs) error { // Setup networking func setupNetworking(args *DockerInitArgs) error { - if args.gateway == "" { - return nil - } + if args.ip != "" { + // eth0 + iface, err := net.InterfaceByName("eth0") + if err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } + ip, ipNet, err := net.ParseCIDR(args.ip) + if err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } + if err := netlink.NetworkLinkAddIp(iface, ip, ipNet); err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } + if err := netlink.NetworkLinkUp(iface); err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } - ip := net.ParseIP(args.gateway) - if ip == nil { - return fmt.Errorf("Unable to set up networking, %s is not a valid IP", args.gateway) + // loopback + iface, err = net.InterfaceByName("lo") + if err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } + if err := netlink.NetworkLinkUp(iface); err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } } + if args.gateway != "" { + gw := net.ParseIP(args.gateway) + if gw == nil { + return fmt.Errorf("Unable to set up networking, %s is not a valid gateway IP", args.gateway) + } - if err := netlink.AddDefaultGw(ip); err != nil { - return fmt.Errorf("Unable to set up networking: %v", err) + if err := netlink.AddDefaultGw(gw); err != nil { + return fmt.Errorf("Unable to set up networking: %v", err) + } } return nil @@ -199,6 +224,7 @@ func SysInit() { // Get cmdline arguments user := flag.String("u", "", "username or uid") gateway := flag.String("g", "", "gateway address") + ip := flag.String("i", "", "ip address") workDir := flag.String("w", "", "workdir") privileged := flag.Bool("privileged", false, "privileged mode") flag.Parse() @@ -216,6 +242,7 @@ func SysInit() { args := &DockerInitArgs{ user: *user, gateway: *gateway, + ip: *ip, workDir: *workDir, privileged: *privileged, env: env, From e8772943215fff3e17642ad410e4815e40e96b8b Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 8 Nov 2013 16:15:11 -0600 Subject: [PATCH 088/105] dockerinit: propagate "container" env variable from lxc Lxc (and libvirt) already set the "container" env variable appropriately[1], so just use that. [1] http://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ --- container.go | 1 - sysinit/sysinit.go | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index 2307948e26..e2fb42c6fc 100644 --- a/container.go +++ b/container.go @@ -586,7 +586,6 @@ func (container *Container) Start() (err error) { env := []string{ "HOME=/", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "container=lxc", "HOSTNAME=" + container.Config.Hostname, } diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 220f446d62..bc708b312a 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -191,6 +191,9 @@ func SysInit() { log.Fatalf("Unable to unmarshal environment variables: %v", err) } + // Propagate the plugin-specific container env variable + env = append(env, "container="+os.Getenv("container")) + args := &DockerInitArgs{ user: *user, gateway: *gateway, From a68d7f3d70ab1638fd5b63a3026b211cb4e7a65e Mon Sep 17 00:00:00 2001 From: WarheadsSE Date: Fri, 13 Dec 2013 10:47:19 -0500 Subject: [PATCH 089/105] Add -bip flag: allow specification of dynamic bridge IP via CIDR e.g.: ``` docker -d -bip "10.10.0.1/16" ``` If set and valid, use provided in place of trial and error from pre-defined array in network.go. Mutually exclusive of -b option. --- config.go | 2 ++ docker/docker.go | 6 ++++++ docs/sources/commandline/cli.rst | 1 + network.go | 33 ++++++++++++++++++++------------ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/config.go b/config.go index 51d435b841..1f743f6290 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ type DaemonConfig struct { Dns []string EnableIptables bool BridgeIface string + BridgeIp string DefaultIp net.IP InterContainerCommunication bool GraphDriver string @@ -36,6 +37,7 @@ func ConfigFromJob(job *engine.Job) *DaemonConfig { } else { config.BridgeIface = DefaultNetworkBridge } + config.BridgeIp = job.Getenv("BridgeIp") config.DefaultIp = net.ParseIP(job.Getenv("DefaultIp")) config.InterContainerCommunication = job.GetenvBool("InterContainerCommunication") config.GraphDriver = job.Getenv("GraphDriver") diff --git a/docker/docker.go b/docker/docker.go index 6e243f157f..3ee7be9876 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -30,6 +30,7 @@ func main() { flDebug = flag.Bool("D", false, "Enable debug mode") flAutoRestart = flag.Bool("r", true, "Restart previously running containers") bridgeName = flag.String("b", "", "Attach containers to a pre-existing network bridge; use 'none' to disable container networking") + bridgeIp = flag.String("bip", "", "Use this CIDR notation address for the network bridge's IP, not compatible with -b") pidfile = flag.String("p", "/var/run/docker.pid", "Path to use for daemon PID file") flRoot = flag.String("g", "/var/lib/docker", "Path to use as the root of the docker runtime") flEnableCors = flag.Bool("api-enable-cors", false, "Enable CORS headers in the remote API") @@ -54,6 +55,10 @@ func main() { flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) } + if *bridgeName != "" && *bridgeIp != "" { + log.Fatal("You specified -b & -bip, mutually exclusive options. Please specify only one.") + } + if *flDebug { os.Setenv("DEBUG", "1") } @@ -77,6 +82,7 @@ func main() { job.SetenvList("Dns", flDns.GetAll()) job.SetenvBool("EnableIptables", *flEnableIptables) job.Setenv("BridgeIface", *bridgeName) + job.Setenv("BridgeIp", *bridgeIp) job.Setenv("DefaultIp", *flDefaultIp) job.SetenvBool("InterContainerCommunication", *flInterContainerComm) job.Setenv("GraphDriver", *flGraphDriver) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 5cbd28c951..68d18b58bf 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -30,6 +30,7 @@ To list available commands, either run ``docker`` with no parameters or execute -H=[unix:///var/run/docker.sock]: Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise -api-enable-cors=false: Enable CORS headers in the remote API -b="": Attach containers to a pre-existing network bridge; use 'none' to disable container networking + -bip="": Use the provided CIDR notation address for the dynamically created bridge (docker0); Mutually exclusive of -b -d=false: Enable daemon mode -dns="": Force docker to use specific DNS servers -g="/var/lib/docker": Path to use as the root of the docker runtime diff --git a/network.go b/network.go index a230356a7d..7999826da1 100644 --- a/network.go +++ b/network.go @@ -118,6 +118,7 @@ func CreateBridgeIface(config *DaemonConfig) error { "192.168.44.1/24", } + nameservers := []string{} resolvConf, _ := utils.GetResolvConf() // we don't check for an error here, because we don't really care @@ -129,22 +130,30 @@ func CreateBridgeIface(config *DaemonConfig) error { } var ifaceAddr string - for _, addr := range addrs { - _, dockerNetwork, err := net.ParseCIDR(addr) + if len(config.BridgeIp) != 0 { + _, _, err := net.ParseCIDR(config.BridgeIp) if err != nil { return err } - routes, err := netlink.NetworkGetRoutes() - if err != nil { - return err - } - if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { - if err := checkNameserverOverlaps(nameservers, dockerNetwork); err == nil { - ifaceAddr = addr - break + ifaceAddr = config.BridgeIp + } else { + for _, addr := range addrs { + _, dockerNetwork, err := net.ParseCIDR(addr) + if err != nil { + return err + } + routes, err := netlink.NetworkGetRoutes() + if err != nil { + return err + } + if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { + if err := checkNameserverOverlaps(nameservers, dockerNetwork); err == nil { + ifaceAddr = addr + break + } + } else { + utils.Debugf("%s: %s", addr, err) } - } else { - utils.Debugf("%s: %s", addr, err) } } if ifaceAddr == "" { From a530b8d9818126e04207de619af8578a4ec4c590 Mon Sep 17 00:00:00 2001 From: Rodrigo Vaz Date: Fri, 13 Dec 2013 16:39:49 -0200 Subject: [PATCH 090/105] fix #3141 Bridge creation when ipv6 is not enabled --- network.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/network.go b/network.go index a230356a7d..3e5f635076 100644 --- a/network.go +++ b/network.go @@ -178,7 +178,11 @@ func CreateBridgeIface(config *DaemonConfig) error { func createBridgeIface(name string) error { s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_IP) if err != nil { - return fmt.Errorf("Error creating bridge creation socket: %s", err) + utils.Debugf("Bridge socket creation failed IPv6 probably not enabled: %v", err) + s, err = syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_IP) + if err != nil { + return fmt.Errorf("Error creating bridge creation socket: %s", err) + } } defer syscall.Close(s) From 59dc2876a7d2ac0e231d7d97511e572a1d739136 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 8 Dec 2013 20:20:55 -0700 Subject: [PATCH 091/105] Add new cover bundlescript for giving a nice report across all the coverprofiles generated by the test scripts --- Dockerfile | 3 +++ hack/make.sh | 16 +++++++++++++++- hack/make/cover | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 hack/make/cover diff --git a/Dockerfile b/Dockerfile index cc5a19276f..b06c6553e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,9 @@ RUN git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /u RUN cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# Grab Go's cover tool for dead-simple code coverage testing +RUN go get code.google.com/p/go.tools/cmd/cover + VOLUME /var/lib/docker WORKDIR /go/src/github.com/dotcloud/docker diff --git a/hack/make.sh b/hack/make.sh index 7b39f3161c..91f641af89 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -39,6 +39,7 @@ DEFAULT_BUNDLES=( dynbinary dyntest dyntest-integration + cover tgz ubuntu ) @@ -64,6 +65,11 @@ LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w' LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' BUILDFLAGS='-tags netgo' +HAVE_GO_TEST_COVER= +if go help testflag | grep -q -- -cover; then + HAVE_GO_TEST_COVER=1 +fi + # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. # @@ -71,6 +77,14 @@ BUILDFLAGS='-tags netgo' # go_test_dir() { dir=$1 + testcover=() + if [ "$HAVE_GO_TEST_COVER" ]; then + # if our current go install has -cover, we want to use it :) + mkdir -p "$DEST/coverprofiles" + coverprofile="docker${dir#.}" + coverprofile="$DEST/coverprofiles/${coverprofile//\//-}" + testcover=( -cover -coverprofile "$coverprofile" ) + fi ( # we run "go test -i" ouside the "set -x" to provde cleaner output cd "$dir" go test -i -ldflags "$LDFLAGS" $BUILDFLAGS @@ -78,7 +92,7 @@ go_test_dir() { ( set -x cd "$dir" - go test -ldflags "$LDFLAGS" $BUILDFLAGS $TESTFLAGS + go test ${testcover[@]} -ldflags "$LDFLAGS" $BUILDFLAGS $TESTFLAGS ) } diff --git a/hack/make/cover b/hack/make/cover new file mode 100644 index 0000000000..6dc71d1c7e --- /dev/null +++ b/hack/make/cover @@ -0,0 +1,21 @@ +#!/bin/bash + +DEST="$1" + +bundle_cover() { + coverprofiles=( "$DEST/../"*"/coverprofiles/"* ) + for p in "${coverprofiles[@]}"; do + echo + ( + set -x + go tool cover -func="$p" + ) + done +} + +if [ "$HAVE_GO_TEST_COVER" ]; then + bundle_cover 2>&1 | tee "$DEST/report.log" +else + echo >&2 'warning: the current version of go does not support -cover' + echo >&2 ' skipping test coverage report' +fi From f63cdf0260cf6287d28a589a79d3f947def6a569 Mon Sep 17 00:00:00 2001 From: Joseph Hager Date: Thu, 12 Dec 2013 16:34:26 -0500 Subject: [PATCH 092/105] Validate container names on creation. Fixes #3138 Move valid container name regex to the top of the file Added hyphen as a valid rune in container names. Remove group in valid container name regex. --- runtime.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime.go b/runtime.go index fbf0facd7d..ca2a135cb6 100644 --- a/runtime.go +++ b/runtime.go @@ -18,6 +18,7 @@ import ( "os" "os/exec" "path" + "regexp" "sort" "strings" "sync" @@ -27,7 +28,10 @@ import ( // Set the max depth to the aufs restriction const MaxImageDepth = 42 -var defaultDns = []string{"8.8.8.8", "8.8.4.4"} +var ( + defaultDns = []string{"8.8.8.8", "8.8.4.4"} + validContainerName = regexp.MustCompile(`^/?[a-zA-Z0-9_-]+$`) +) type Capabilities struct { MemoryLimit bool @@ -418,7 +422,12 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin if err != nil { name = utils.TruncateID(id) } + } else { + if !validContainerName.MatchString(name) { + return nil, nil, fmt.Errorf("Invalid container name (%s), only [a-zA-Z0-9_-] are allowed", name) + } } + if name[0] != '/' { name = "/" + name } From 1e85aabf710b2bc8d47027973fe19f4f10a5ba1e Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Fri, 13 Dec 2013 11:42:58 -0800 Subject: [PATCH 093/105] Fix #1695 by adding more about logging. --- docs/sources/commandline/cli.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 4ef3671186..276e689e8b 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -794,6 +794,15 @@ Known Issues (kill) Fetch the logs of a container +``docker logs`` is a convenience which batch-retrieves whatever logs +are present at the time of execution. This does not guarantee +execution order when combined with a ``docker run`` (i.e. your run may +not have generated any logs at the time you execute ``docker logs``). + +``docker logs -f`` combines ``docker logs`` and ``docker attach``: it +will first return all logs from the beginning and then continue +streaming new output from the container's stdout and stderr. + .. _cli_port: From 73e8a39ff219833ebb28c12dc318466e3551b5bd Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 11 Dec 2013 18:25:30 -0800 Subject: [PATCH 094/105] move resize to job --- api.go | 11 +---------- server.go | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/api.go b/api.go index f7a7ac91a4..faa6d3695f 100644 --- a/api.go +++ b/api.go @@ -717,19 +717,10 @@ func postContainersResize(srv *Server, version float64, w http.ResponseWriter, r if err := parseForm(r); err != nil { return err } - height, err := strconv.Atoi(r.Form.Get("h")) - if err != nil { - return err - } - width, err := strconv.Atoi(r.Form.Get("w")) - if err != nil { - return err - } if vars == nil { return fmt.Errorf("Missing parameter") } - name := vars["name"] - if err := srv.ContainerResize(name, height, width); err != nil { + if err := srv.Eng.Job("resize", vars["name"], r.Form.Get("h"), r.Form.Get("w")).Run(); err != nil { return err } return nil diff --git a/server.go b/server.go index 3657c67a94..1d716114eb 100644 --- a/server.go +++ b/server.go @@ -99,6 +99,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("resize", srv.ContainerResize); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -1750,11 +1754,31 @@ func (srv *Server) ContainerWait(job *engine.Job) engine.Status { return engine.StatusErr } -func (srv *Server) ContainerResize(name string, h, w int) error { - if container := srv.runtime.Get(name); container != nil { - return container.Resize(h, w) +func (srv *Server) ContainerResize(job *engine.Job) engine.Status { + if len(job.Args) != 3 { + job.Errorf("Not enough arguments. Usage: %s CONTAINER HEIGHT WIDTH\n", job.Name) + return engine.StatusErr } - return fmt.Errorf("No such container: %s", name) + name := job.Args[0] + height, err := strconv.Atoi(job.Args[1]) + if err != nil { + job.Error(err) + return engine.StatusErr + } + width, err := strconv.Atoi(job.Args[2]) + if err != nil { + job.Error(err) + return engine.StatusErr + } + if container := srv.runtime.Get(name); container != nil { + if err := container.Resize(height, width); err != nil { + job.Error(err) + return engine.StatusErr + } + return engine.StatusOK + } + job.Errorf("No such container: %s", name) + return engine.StatusErr } func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, stderr bool, inStream io.ReadCloser, outStream, errStream io.Writer) error { From 4975c1b54980517b5527278e637f3c05dd6a32db Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 13 Dec 2013 13:51:20 -0800 Subject: [PATCH 095/105] Temporarily remve @shykes from engine/MAINTAINERS --- engine/MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/MAINTAINERS b/engine/MAINTAINERS index e1c6f2ccfc..354798f72e 100644 --- a/engine/MAINTAINERS +++ b/engine/MAINTAINERS @@ -1 +1 @@ -Solomon Hykes +#Solomon Hykes Temporarily unavailable From 930ec9f52ceceabcb72589ac7f20e1e8bbccbed5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 11 Dec 2013 17:03:48 -0800 Subject: [PATCH 096/105] move commit to job --- api.go | 19 ++++++++++++------- engine/env.go | 22 +++++++++++++++------- engine/job.go | 4 ++++ integration/server_test.go | 22 ++++++++++++++++------ server.go | 29 ++++++++++++++++++++++++----- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index f7e37ef07c..4e8e3330c7 100644 --- a/api.go +++ b/api.go @@ -377,13 +377,18 @@ func postCommit(srv *Server, version float64, w http.ResponseWriter, r *http.Req if err := json.NewDecoder(r.Body).Decode(config); err != nil && err != io.EOF { utils.Errorf("%s", err) } - repo := r.Form.Get("repo") - tag := r.Form.Get("tag") - container := r.Form.Get("container") - author := r.Form.Get("author") - comment := r.Form.Get("comment") - id, err := srv.ContainerCommit(container, repo, tag, author, comment, config) - if err != nil { + + job := srv.Eng.Job("commit") + job.Setenv("repo", r.Form.Get("repo")) + job.Setenv("tag", r.Form.Get("tag")) + job.Setenv("container", r.Form.Get("container")) + job.Setenv("author", r.Form.Get("author")) + job.Setenv("comment", r.Form.Get("comment")) + job.SetenvJson("config", config) + + var id string + job.Stdout.AddString(&id) + if err := job.Run(); err != nil { return err } diff --git a/engine/env.go b/engine/env.go index 5e4f59395a..588cdeab2a 100644 --- a/engine/env.go +++ b/engine/env.go @@ -1,17 +1,16 @@ package engine import ( - "strings" - "io" - "encoding/json" - "strconv" "bytes" + "encoding/json" "fmt" + "io" + "strconv" + "strings" ) type Env []string - func (env *Env) Get(key string) (value string) { // FIXME: use Map() for _, kv := range *env { @@ -44,7 +43,6 @@ func (env *Env) GetBool(key string) (value bool) { return true } - func (env *Env) SetBool(key string, value bool) { if value { env.Set(key, "1") @@ -79,6 +77,17 @@ func (env *Env) GetList(key string) []string { return l } +func (env *Env) GetJson(key string) interface{} { + sval := env.Get(key) + if sval == "" { + return nil + } + var m interface{} + //Discard error on purpose + json.Unmarshal([]byte(sval), &m) + return m +} + func (env *Env) SetJson(key string, value interface{}) error { sval, err := json.Marshal(value) if err != nil { @@ -218,4 +227,3 @@ func (env *Env) Map() map[string]string { } return m } - diff --git a/engine/job.go b/engine/job.go index 465086af91..c7b510d47b 100644 --- a/engine/job.go +++ b/engine/job.go @@ -126,6 +126,10 @@ func (job *Job) GetenvList(key string) []string { return job.env.GetList(key) } +func (job *Job) GetenvJson(key string) interface{} { + return job.env.GetJson(key) +} + func (job *Job) SetenvJson(key string, value interface{}) error { return job.env.SetJson(key, value) } diff --git a/integration/server_test.go b/integration/server_test.go index 7fb7611192..ab9e39c2e0 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -146,7 +146,6 @@ func TestCreateRmVolumes(t *testing.T) { func TestCommit(t *testing.T) { eng := NewTestEngine(t) - srv := mkServerFromEngine(eng, t) defer mkRuntimeFromEngine(eng, t).Nuke() config, _, _, err := docker.ParseRun([]string{unitTestImageID, "/bin/cat"}, nil) @@ -156,7 +155,12 @@ func TestCommit(t *testing.T) { id := createTestContainer(eng, config, t) - if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { + job := eng.Job("commit") + job.Setenv("container", id) + job.Setenv("repo", "testrepo") + job.Setenv("tag", "testtag") + job.SetenvJson("config", config) + if err := job.Run(); err != nil { t.Fatal(err) } } @@ -264,8 +268,12 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil) - if err != nil { + job = eng.Job("commit") + job.Setenv("container", containerID) + job.Setenv("repo", "test") + var imageID string + job.Stdout.AddString(&imageID) + if err := job.Run(); err != nil { t.Fatal(err) } @@ -288,8 +296,10 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - _, err = srv.ContainerCommit(containerID, "test", "", "", "", nil) - if err != nil { + job = eng.Job("commit") + job.Setenv("container", containerID) + job.Setenv("repo", "test") + if err := job.Run(); err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index 63643cb20e..4ae34c4290 100644 --- a/server.go +++ b/server.go @@ -107,6 +107,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("commit", srv.ContainerCommit); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -769,16 +773,31 @@ func createAPIContainer(names []string, container *Container, size bool, runtime } return c } -func (srv *Server) ContainerCommit(name, repo, tag, author, comment string, config *Config) (string, error) { +func (srv *Server) ContainerCommit(job *engine.Job) engine.Status { + if len(job.Args) != 0 { + job.Errorf("Usage: %s\n", job.Name) + return engine.StatusErr + } + name := job.Getenv("container") + container := srv.runtime.Get(name) if container == nil { - return "", fmt.Errorf("No such container: %s", name) + job.Errorf("No such container: %s", name) + return engine.StatusErr } - img, err := srv.runtime.Commit(container, repo, tag, comment, author, config) + var config *Config + iConfig, ok := job.GetenvJson("config").(Config) + if ok { + config = &iConfig + } + + img, err := srv.runtime.Commit(container, job.Getenv("repo"), job.Getenv("tag"), job.Getenv("comment"), job.Getenv("author"), config) if err != nil { - return "", err + job.Error(err) + return engine.StatusErr } - return img.ID, err + job.Printf("%s\n", img.ID) + return engine.StatusOK } func (srv *Server) ImageTag(job *engine.Job) engine.Status { From 4b5ceb0f24285718620465bb4073269d6fb8097b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 13 Dec 2013 14:29:27 -0800 Subject: [PATCH 097/105] use args --- api.go | 3 +-- integration/server_test.go | 9 +++------ server.go | 6 +++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index 4e8e3330c7..0999ebaa03 100644 --- a/api.go +++ b/api.go @@ -378,10 +378,9 @@ func postCommit(srv *Server, version float64, w http.ResponseWriter, r *http.Req utils.Errorf("%s", err) } - job := srv.Eng.Job("commit") + job := srv.Eng.Job("commit", r.Form.Get("container")) job.Setenv("repo", r.Form.Get("repo")) job.Setenv("tag", r.Form.Get("tag")) - job.Setenv("container", r.Form.Get("container")) job.Setenv("author", r.Form.Get("author")) job.Setenv("comment", r.Form.Get("comment")) job.SetenvJson("config", config) diff --git a/integration/server_test.go b/integration/server_test.go index ab9e39c2e0..90752e23ef 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -155,8 +155,7 @@ func TestCommit(t *testing.T) { id := createTestContainer(eng, config, t) - job := eng.Job("commit") - job.Setenv("container", id) + job := eng.Job("commit", id) job.Setenv("repo", "testrepo") job.Setenv("tag", "testtag") job.SetenvJson("config", config) @@ -268,8 +267,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - job = eng.Job("commit") - job.Setenv("container", containerID) + job = eng.Job("commit", containerID) job.Setenv("repo", "test") var imageID string job.Stdout.AddString(&imageID) @@ -296,8 +294,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - job = eng.Job("commit") - job.Setenv("container", containerID) + job = eng.Job("commit", containerID) job.Setenv("repo", "test") if err := job.Run(); err != nil { t.Fatal(err) diff --git a/server.go b/server.go index 4ae34c4290..e7bb73d4ed 100644 --- a/server.go +++ b/server.go @@ -774,11 +774,11 @@ func createAPIContainer(names []string, container *Container, size bool, runtime return c } func (srv *Server) ContainerCommit(job *engine.Job) engine.Status { - if len(job.Args) != 0 { - job.Errorf("Usage: %s\n", job.Name) + if len(job.Args) != 1 { + job.Errorf("Not enough arguments. Usage: %s CONTAINER\n", job.Name) return engine.StatusErr } - name := job.Getenv("container") + name := job.Args[0] container := srv.runtime.Get(name) if container == nil { From d5f5ecb658c7a6a0f04939061494c9d404d393ce Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 13 Dec 2013 15:01:54 -0800 Subject: [PATCH 098/105] improve GetenvJson --- engine/env.go | 7 ++----- engine/job.go | 4 ++-- server.go | 10 +++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/engine/env.go b/engine/env.go index 588cdeab2a..81c76d5adb 100644 --- a/engine/env.go +++ b/engine/env.go @@ -77,15 +77,12 @@ func (env *Env) GetList(key string) []string { return l } -func (env *Env) GetJson(key string) interface{} { +func (env *Env) GetJson(key string, iface interface{}) error { sval := env.Get(key) if sval == "" { return nil } - var m interface{} - //Discard error on purpose - json.Unmarshal([]byte(sval), &m) - return m + return json.Unmarshal([]byte(sval), iface) } func (env *Env) SetJson(key string, value interface{}) error { diff --git a/engine/job.go b/engine/job.go index c7b510d47b..e960cffda3 100644 --- a/engine/job.go +++ b/engine/job.go @@ -126,8 +126,8 @@ func (job *Job) GetenvList(key string) []string { return job.env.GetList(key) } -func (job *Job) GetenvJson(key string) interface{} { - return job.env.GetJson(key) +func (job *Job) GetenvJson(key string, iface interface{}) error { + return job.env.GetJson(key, iface) } func (job *Job) SetenvJson(key string, value interface{}) error { diff --git a/server.go b/server.go index e7bb73d4ed..ee1ca939e3 100644 --- a/server.go +++ b/server.go @@ -785,13 +785,13 @@ func (srv *Server) ContainerCommit(job *engine.Job) engine.Status { job.Errorf("No such container: %s", name) return engine.StatusErr } - var config *Config - iConfig, ok := job.GetenvJson("config").(Config) - if ok { - config = &iConfig + var config Config + if err := job.GetenvJson("config", &config); err != nil { + job.Error(err) + return engine.StatusErr } - img, err := srv.runtime.Commit(container, job.Getenv("repo"), job.Getenv("tag"), job.Getenv("comment"), job.Getenv("author"), config) + img, err := srv.runtime.Commit(container, job.Getenv("repo"), job.Getenv("tag"), job.Getenv("comment"), job.Getenv("author"), &config) if err != nil { job.Error(err) return engine.StatusErr From 51e2c1794b50295607c6eddb29edf39d40816a88 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 11 Dec 2013 10:35:21 -0800 Subject: [PATCH 099/105] move docker info to the job api --- api.go | 3 +- api_params.go | 17 ----------- commands.go | 66 +++++++++++++++++++++++++---------------- integration/api_test.go | 12 +++++--- server.go | 44 ++++++++++++++++----------- 5 files changed, 77 insertions(+), 65 deletions(-) diff --git a/api.go b/api.go index 0999ebaa03..819926692b 100644 --- a/api.go +++ b/api.go @@ -216,7 +216,8 @@ func getImagesViz(srv *Server, version float64, w http.ResponseWriter, r *http.R } func getInfo(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - return writeJSON(w, http.StatusOK, srv.DockerInfo()) + srv.Eng.ServeHTTP(w, r) + return nil } func getEvents(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/api_params.go b/api_params.go index 411cafae2e..fa9eab0c15 100644 --- a/api_params.go +++ b/api_params.go @@ -29,23 +29,6 @@ type ( VirtualSize int64 } - APIInfo struct { - Debug bool - Containers int - Images int - Driver string `json:",omitempty"` - DriverStatus [][2]string `json:",omitempty"` - NFd int `json:",omitempty"` - NGoroutines int `json:",omitempty"` - MemoryLimit bool `json:",omitempty"` - SwapLimit bool `json:",omitempty"` - IPv4Forwarding bool `json:",omitempty"` - LXCVersion string `json:",omitempty"` - NEventsListener int `json:",omitempty"` - KernelVersion string `json:",omitempty"` - IndexServerAddress string `json:",omitempty"` - } - APITop struct { Titles []string Processes [][]string diff --git a/commands.go b/commands.go index 1ec7774243..982d4b3d84 100644 --- a/commands.go +++ b/commands.go @@ -433,42 +433,58 @@ func (cli *DockerCli) CmdInfo(args ...string) error { return err } - var out APIInfo - if err := json.Unmarshal(body, &out); err != nil { + out := engine.NewOutput() + remoteInfo, err := out.AddEnv() + if err != nil { return err } - fmt.Fprintf(cli.out, "Containers: %d\n", out.Containers) - fmt.Fprintf(cli.out, "Images: %d\n", out.Images) - fmt.Fprintf(cli.out, "Driver: %s\n", out.Driver) - for _, pair := range out.DriverStatus { - fmt.Fprintf(cli.out, " %s: %s\n", pair[0], pair[1]) - } - if out.Debug || os.Getenv("DEBUG") != "" { - fmt.Fprintf(cli.out, "Debug mode (server): %v\n", out.Debug) - fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "") - fmt.Fprintf(cli.out, "Fds: %d\n", out.NFd) - fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines) - fmt.Fprintf(cli.out, "LXC Version: %s\n", out.LXCVersion) - fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) - fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) + if _, err := out.Write(body); err != nil { + utils.Errorf("Error reading remote info: %s\n", err) + return err } + out.Close() - if len(out.IndexServerAddress) != 0 { - cli.LoadConfigFile() - u := cli.configFile.Configs[out.IndexServerAddress].Username - if len(u) > 0 { - fmt.Fprintf(cli.out, "Username: %v\n", u) - fmt.Fprintf(cli.out, "Registry: %v\n", out.IndexServerAddress) + fmt.Fprintf(cli.out, "Containers: %d\n", remoteInfo.GetInt("Containers")) + fmt.Fprintf(cli.out, "Images: %d\n", remoteInfo.GetInt("Images")) + fmt.Fprintf(cli.out, "Driver: %s\n", remoteInfo.Get("Driver")) + + //FIXME:Cleanup this mess + DriverStatus := remoteInfo.GetJson("DriverStatus") + if DriverStatus != nil { + if tab, ok := DriverStatus.([]interface{}); ok { + for _, line := range tab { + if pair, ok := line.([]interface{}); ok { + fmt.Fprintf(cli.out, " %s: %s\n", pair[0], pair[1]) + } + } } } - if !out.MemoryLimit { + if remoteInfo.GetBool("Debug") || os.Getenv("DEBUG") != "" { + fmt.Fprintf(cli.out, "Debug mode (server): %v\n", remoteInfo.GetBool("Debug")) + fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "") + fmt.Fprintf(cli.out, "Fds: %d\n", remoteInfo.GetInt("NFd")) + fmt.Fprintf(cli.out, "Goroutines: %d\n", remoteInfo.GetInt("NGoroutines")) + fmt.Fprintf(cli.out, "LXC Version: %s\n", remoteInfo.Get("LXCVersion")) + fmt.Fprintf(cli.out, "EventsListeners: %d\n", remoteInfo.GetInt("NEventsListener")) + fmt.Fprintf(cli.out, "Kernel Version: %s\n", remoteInfo.Get("KernelVersion")) + } + + if len(remoteInfo.GetList("IndexServerAddress")) != 0 { + cli.LoadConfigFile() + u := cli.configFile.Configs[remoteInfo.Get("IndexServerAddress")].Username + if len(u) > 0 { + fmt.Fprintf(cli.out, "Username: %v\n", u) + fmt.Fprintf(cli.out, "Registry: %v\n", remoteInfo.GetList("IndexServerAddress")) + } + } + if !remoteInfo.GetBool("MemoryLimit") { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") } - if !out.SwapLimit { + if !remoteInfo.GetBool("SwapLimit") { fmt.Fprintf(cli.err, "WARNING: No swap limit support\n") } - if !out.IPv4Forwarding { + if !remoteInfo.GetBool("IPv4Forwarding") { fmt.Fprintf(cli.err, "WARNING: IPv4 forwarding is disabled.\n") } return nil diff --git a/integration/api_test.go b/integration/api_test.go index eed17c9084..90a63ba554 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -72,13 +72,17 @@ func TestGetInfo(t *testing.T) { } assertHttpNotError(r, t) - infos := &docker.APIInfo{} - err = json.Unmarshal(r.Body.Bytes(), infos) + out := engine.NewOutput() + i, err := out.AddEnv() if err != nil { t.Fatal(err) } - if infos.Images != len(initialImages) { - t.Errorf("Expected images: %d, %d found", len(initialImages), infos.Images) + if _, err := io.Copy(out, r.Body); err != nil { + t.Fatal(err) + } + out.Close() + if images := i.GetInt("Images"); images != int64(len(initialImages)) { + t.Errorf("Expected images: %d, %d found", len(initialImages), images) } } diff --git a/server.go b/server.go index ee1ca939e3..1a168fe591 100644 --- a/server.go +++ b/server.go @@ -111,6 +111,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("info", srv.DockerInfo); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -610,13 +614,13 @@ func (srv *Server) Images(all bool, filter string) ([]APIImages, error) { return outs, nil } -func (srv *Server) DockerInfo() *APIInfo { +func (srv *Server) DockerInfo(job *engine.Job) engine.Status { images, _ := srv.runtime.graph.Map() - var imgcount int + var imgcount int64 if images == nil { imgcount = 0 } else { - imgcount = len(images) + imgcount = int64(len(images)) } lxcVersion := "" if output, err := exec.Command("lxc-version").CombinedOutput(); err == nil { @@ -630,22 +634,26 @@ func (srv *Server) DockerInfo() *APIInfo { kernelVersion = kv.String() } - return &APIInfo{ - Containers: len(srv.runtime.List()), - Images: imgcount, - Driver: srv.runtime.driver.String(), - DriverStatus: srv.runtime.driver.Status(), - MemoryLimit: srv.runtime.capabilities.MemoryLimit, - SwapLimit: srv.runtime.capabilities.SwapLimit, - IPv4Forwarding: !srv.runtime.capabilities.IPv4ForwardingDisabled, - Debug: os.Getenv("DEBUG") != "", - NFd: utils.GetTotalUsedFds(), - NGoroutines: runtime.NumGoroutine(), - LXCVersion: lxcVersion, - NEventsListener: len(srv.events), - KernelVersion: kernelVersion, - IndexServerAddress: auth.IndexServerAddress(), + v := &engine.Env{} + v.SetInt("Containers", int64(len(srv.runtime.List()))) + v.SetInt("Images", imgcount) + v.Set("Driver", srv.runtime.driver.String()) + v.SetJson("DriverStatus", srv.runtime.driver.Status()) + v.SetBool("MemoryLimit", srv.runtime.capabilities.MemoryLimit) + v.SetBool("SwapLimit", srv.runtime.capabilities.SwapLimit) + v.SetBool("IPv4Forwarding", !srv.runtime.capabilities.IPv4ForwardingDisabled) + v.SetBool("Debug", os.Getenv("DEBUG") != "") + v.SetInt("NFd", int64(utils.GetTotalUsedFds())) + v.SetInt("NGoroutines", int64(runtime.NumGoroutine())) + v.Set("LXCVersion", lxcVersion) + v.SetInt("NEventsListener", int64(len(srv.events))) + v.Set("KernelVersion", kernelVersion) + v.Set("IndexServerAddress", auth.IndexServerAddress()) + if _, err := v.WriteTo(job.Stdout); err != nil { + job.Error(err) + return engine.StatusErr } + return engine.StatusOK } func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { From 85b9338205da0c8f1d62f277db342cf4b9feaf13 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 12 Dec 2013 13:35:50 -0800 Subject: [PATCH 100/105] add GetenvInt64 ans SetenvInt64 --- commands.go | 17 ++++++----------- engine/env.go | 14 +++++++++++--- engine/job.go | 12 ++++++++++-- integration/api_test.go | 2 +- server.go | 12 ++++++------ 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/commands.go b/commands.go index 982d4b3d84..df49b54492 100644 --- a/commands.go +++ b/commands.go @@ -448,17 +448,12 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Containers: %d\n", remoteInfo.GetInt("Containers")) fmt.Fprintf(cli.out, "Images: %d\n", remoteInfo.GetInt("Images")) fmt.Fprintf(cli.out, "Driver: %s\n", remoteInfo.Get("Driver")) - - //FIXME:Cleanup this mess - DriverStatus := remoteInfo.GetJson("DriverStatus") - if DriverStatus != nil { - if tab, ok := DriverStatus.([]interface{}); ok { - for _, line := range tab { - if pair, ok := line.([]interface{}); ok { - fmt.Fprintf(cli.out, " %s: %s\n", pair[0], pair[1]) - } - } - } + var driverStatus [][2]string + if err := remoteInfo.GetJson("DriverStatus", &driverStatus); err != nil { + return err + } + for _, pair := range driverStatus { + fmt.Fprintf(cli.out, " %s: %s\n", pair[0], pair[1]) } if remoteInfo.GetBool("Debug") || os.Getenv("DEBUG") != "" { fmt.Fprintf(cli.out, "Debug mode (server): %v\n", remoteInfo.GetBool("Debug")) diff --git a/engine/env.go b/engine/env.go index 81c76d5adb..a65c8438d2 100644 --- a/engine/env.go +++ b/engine/env.go @@ -51,7 +51,11 @@ func (env *Env) SetBool(key string, value bool) { } } -func (env *Env) GetInt(key string) int64 { +func (env *Env) GetInt(key string) int { + return int(env.GetInt64(key)) +} + +func (env *Env) GetInt64(key string) int64 { s := strings.Trim(env.Get(key), " \t") val, err := strconv.ParseInt(s, 10, 64) if err != nil { @@ -60,7 +64,11 @@ func (env *Env) GetInt(key string) int64 { return val } -func (env *Env) SetInt(key string, value int64) { +func (env *Env) SetInt(key string, value int) { + env.Set(key, fmt.Sprintf("%d", value)) +} + +func (env *Env) SetInt64(key string, value int64) { env.Set(key, fmt.Sprintf("%d", value)) } @@ -145,7 +153,7 @@ func (env *Env) SetAuto(k string, v interface{}) { // encoding/json decodes integers to float64, but cannot encode them back. // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) if fval, ok := v.(float64); ok { - env.SetInt(k, int64(fval)) + env.SetInt64(k, int64(fval)) } else if sval, ok := v.(string); ok { env.Set(k, sval) } else if val, err := json.Marshal(v); err == nil { diff --git a/engine/job.go b/engine/job.go index e960cffda3..68b1715d92 100644 --- a/engine/job.go +++ b/engine/job.go @@ -113,11 +113,19 @@ func (job *Job) SetenvBool(key string, value bool) { job.env.SetBool(key, value) } -func (job *Job) GetenvInt(key string) int64 { +func (job *Job) GetenvInt64(key string) int64 { + return job.env.GetInt64(key) +} + +func (job *Job) GetenvInt(key string) int { return job.env.GetInt(key) } -func (job *Job) SetenvInt(key string, value int64) { +func (job *Job) SetenvInt64(key string, value int64) { + job.env.SetInt64(key, value) +} + +func (job *Job) SetenvInt(key string, value int) { job.env.SetInt(key, value) } diff --git a/integration/api_test.go b/integration/api_test.go index 90a63ba554..b635dd81c0 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -81,7 +81,7 @@ func TestGetInfo(t *testing.T) { t.Fatal(err) } out.Close() - if images := i.GetInt("Images"); images != int64(len(initialImages)) { + if images := i.GetInt("Images"); images != len(initialImages) { t.Errorf("Expected images: %d, %d found", len(initialImages), images) } } diff --git a/server.go b/server.go index 1a168fe591..a0e6191090 100644 --- a/server.go +++ b/server.go @@ -616,11 +616,11 @@ func (srv *Server) Images(all bool, filter string) ([]APIImages, error) { func (srv *Server) DockerInfo(job *engine.Job) engine.Status { images, _ := srv.runtime.graph.Map() - var imgcount int64 + var imgcount int if images == nil { imgcount = 0 } else { - imgcount = int64(len(images)) + imgcount = len(images) } lxcVersion := "" if output, err := exec.Command("lxc-version").CombinedOutput(); err == nil { @@ -635,7 +635,7 @@ func (srv *Server) DockerInfo(job *engine.Job) engine.Status { } v := &engine.Env{} - v.SetInt("Containers", int64(len(srv.runtime.List()))) + v.SetInt("Containers", len(srv.runtime.List())) v.SetInt("Images", imgcount) v.Set("Driver", srv.runtime.driver.String()) v.SetJson("DriverStatus", srv.runtime.driver.Status()) @@ -643,10 +643,10 @@ func (srv *Server) DockerInfo(job *engine.Job) engine.Status { v.SetBool("SwapLimit", srv.runtime.capabilities.SwapLimit) v.SetBool("IPv4Forwarding", !srv.runtime.capabilities.IPv4ForwardingDisabled) v.SetBool("Debug", os.Getenv("DEBUG") != "") - v.SetInt("NFd", int64(utils.GetTotalUsedFds())) - v.SetInt("NGoroutines", int64(runtime.NumGoroutine())) + v.SetInt("NFd", utils.GetTotalUsedFds()) + v.SetInt("NGoroutines", runtime.NumGoroutine()) v.Set("LXCVersion", lxcVersion) - v.SetInt("NEventsListener", int64(len(srv.events))) + v.SetInt("NEventsListener", len(srv.events)) v.Set("KernelVersion", kernelVersion) v.Set("IndexServerAddress", auth.IndexServerAddress()) if _, err := v.WriteTo(job.Stdout); err != nil { From 0a3eedd4c9d2bed208a85cd7aefae050a821eb6b Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Sat, 14 Dec 2013 15:29:08 +1000 Subject: [PATCH 101/105] when sharing a /var/lib/docker dir with more than one distribution, an existing lxc-start-unconfined softlink may point to a non-existant path, following that link (as Stat does) will cause the daemon to fail to start --- runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime.go b/runtime.go index a2669343ce..3268892d56 100644 --- a/runtime.go +++ b/runtime.go @@ -868,7 +868,7 @@ func linkLxcStart(root string) error { } targetPath := path.Join(root, "lxc-start-unconfined") - if _, err := os.Stat(targetPath); err != nil && !os.IsNotExist(err) { + if _, err := os.Lstat(targetPath); err != nil && !os.IsNotExist(err) { return err } else if err == nil { if err := os.Remove(targetPath); err != nil { From 12fb50826297c40dcfd047e8635f40c6824797ac Mon Sep 17 00:00:00 2001 From: unclejack Date: Sat, 14 Dec 2013 16:00:52 +0200 Subject: [PATCH 102/105] install vbox guest additions if not latest --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 2d6b488cfe..013027ecde 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -70,7 +70,7 @@ SCRIPT # trigger dkms to build the virtualbox guest module install. $vbox_script = < Date: Sun, 15 Dec 2013 09:17:16 -0500 Subject: [PATCH 103/105] Update readme to mark ZFS driver as Alpha quality. --- contrib/zfs/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/zfs/README.md b/contrib/zfs/README.md index 04d36841a3..84f6296e10 100644 --- a/contrib/zfs/README.md +++ b/contrib/zfs/README.md @@ -11,7 +11,8 @@ branch named [zfs_driver]. # Status -Pre-alpha +Alpha: The code is now capable of creating, running and destroying containers +and images. The code is under development. Contributions in the form of suggestions, code-reviews, and patches are welcome. From 62213ee314d0669a278fae1e3052329f4ca35f75 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 16 Dec 2013 13:29:43 -0800 Subject: [PATCH 104/105] Allow untag operations with no container validation --- integration/server_test.go | 47 +++++++++++++++++++++++++++++ server.go | 61 +++++++++++++++++++++++++------------- 2 files changed, 87 insertions(+), 21 deletions(-) diff --git a/integration/server_test.go b/integration/server_test.go index 90752e23ef..2650311c36 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -409,3 +409,50 @@ func TestImageInsert(t *testing.T) { t.Fatalf("expected no error, but got %v", err) } } + +// Regression test for being able to untag an image with an existing +// container +func TestDeleteTagWithExistingContainers(t *testing.T) { + eng := NewTestEngine(t) + defer nuke(mkRuntimeFromEngine(eng, t)) + + srv := mkServerFromEngine(eng, t) + + // Tag the image + if err := eng.Job("tag", unitTestImageID, "utest", "tag1").Run(); err != nil { + t.Fatal(err) + } + + // Create a container from the image + config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + if err != nil { + t.Fatal(err) + } + + id := createNamedTestContainer(eng, config, t, "testingtags") + if id == "" { + t.Fatal("No id returned") + } + + containers := srv.Containers(true, false, -1, "", "") + + if len(containers) != 1 { + t.Fatalf("Expected 1 container got %d", len(containers)) + } + + // Try to remove the tag + imgs, err := srv.ImageDelete("utest:tag1", true) + if err != nil { + t.Fatal(err) + } + + if len(imgs) != 1 { + t.Fatalf("Should only have deleted one untag %d", len(imgs)) + } + + untag := imgs[0] + + if untag.Untagged != unitTestImageID { + t.Fatalf("Expected %s got %s", unitTestImageID, untag.Untagged) + } +} diff --git a/server.go b/server.go index a0e6191090..9b492a2927 100644 --- a/server.go +++ b/server.go @@ -1550,8 +1550,10 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error { } func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, error) { - imgs := []APIRmi{} - tags := []string{} + var ( + imgs = []APIRmi{} + tags = []string{} + ) //If delete by id, see if the id belong only to one repository if repoName == "" { @@ -1571,6 +1573,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro } else { tags = append(tags, tag) } + //Untag the current image for _, tag := range tags { tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag) @@ -1582,6 +1585,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro srv.LogEvent("untag", img.ID, "") } } + if len(srv.runtime.repositories.ByID()[img.ID]) == 0 { if err := srv.deleteImageAndChildren(img.ID, &imgs, nil); err != nil { if err != ErrImageReferenced { @@ -1597,10 +1601,16 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro } func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { + var ( + repository, tag string + validate = true + ) img, err := srv.runtime.repositories.LookupImage(name) if err != nil { return nil, fmt.Errorf("No such image: %s", name) } + + // FIXME: What does autoPrune mean ? if !autoPrune { if err := srv.runtime.graph.Delete(img.ID); err != nil { return nil, fmt.Errorf("Cannot delete image %s: %s", name, err) @@ -1608,29 +1618,38 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - // Prevent deletion if image is used by a container - for _, container := range srv.runtime.List() { - parent, err := srv.runtime.repositories.LookupImage(container.Image) - if err != nil { - return nil, err - } + if !strings.Contains(img.ID, name) { + repository, tag = utils.ParseRepositoryTag(name) + } - if err := parent.WalkHistory(func(p *Image) error { - if img.ID == p.ID { - return fmt.Errorf("Conflict, cannot delete %s because the container %s is using it", name, container.ID) + // If we have a repo and the image is not referenced anywhere else + // then just perform an untag and do not validate. + // + // i.e. only validate if we are performing an actual delete and not + // an untag op + if repository != "" { + validate = len(srv.runtime.repositories.ByID()[img.ID]) == 1 + } + + if validate { + // Prevent deletion if image is used by a container + for _, container := range srv.runtime.List() { + parent, err := srv.runtime.repositories.LookupImage(container.Image) + if err != nil { + return nil, err + } + + if err := parent.WalkHistory(func(p *Image) error { + if img.ID == p.ID { + return fmt.Errorf("Conflict, cannot delete %s because the container %s is using it", name, container.ID) + } + return nil + }); err != nil { + return nil, err } - return nil - }); err != nil { - return nil, err } } - - if strings.Contains(img.ID, name) { - //delete via ID - return srv.deleteImage(img, "", "") - } - name, tag := utils.ParseRepositoryTag(name) - return srv.deleteImage(img, name, tag) + return srv.deleteImage(img, repository, tag) } func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) { From e960152a1e9064d8c2ae57b9ab2a33d9b27276b9 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 16 Dec 2013 14:50:07 -0800 Subject: [PATCH 105/105] Bump to v0.7.2 --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86e3b88484..060a07c21e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## 0.7.2 (2013-12-16) + +#### Runtime + ++ Validate container names on creation with standard regex +* Increase maximum image depth to 127 from 42 +* Continue to move api endpoints to the job api ++ Add -bip flag to allow specification of dynamic bridge IP via CIDR +- Allow bridge creation when ipv6 is not enabled on certain systems +* Set hostname and IP address from within dockerinit +* Drop capabilities from within dockerinit +- Fix volumes on host when symlink is present the image +- Prevent deletion of image if ANY container is depending on it even if the container is not running +* Update docker push to use new progress display +* Use os.Lstat to allow mounting unix sockets when inspecting volumes +- Adjusted handling of inactive user login +- Add missing defines in devicemapper for older kernels +- Allow untag operations with no container validation +- Add auth config to docker build + +#### Documentation + +* Add more information about Docker logging ++ Add RHEL documentation +* Add a direct example for changing the CMD that is run in a container +* Update Arch installation documentation ++ Add section on Trusted Builds ++ Add Network documentation page + +#### Other + ++ Add new cover bundle for providing code coverage reporting +* Separate integration tests in bundles +* Make Tianon the hack maintainer +* Update mkimage-debootstrap with more tweaks for keeping images small +* Use https to get the install script +* Remove vendored dotcloud/tar now that Go 1.2 has been released + ## 0.7.1 (2013-12-05) #### Documentation diff --git a/VERSION b/VERSION index c0ab427224..7486fdbc50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1-dev +0.7.2