From 821a82ac6ceffd7001b80769ba23b81a7e4a81ad Mon Sep 17 00:00:00 2001 From: Nick Stinemates Date: Fri, 4 Oct 2013 14:17:59 -0700 Subject: [PATCH 01/41] Add Developer Certificate of Origin Text We purposely chose to license Docker under the Apache 2.0 license. This is a well-known license, which is popular for its permissive and flexible properties, while still encouraging a collaborative community. It also makes certain representations with regard to contributions, and the rights given to contributors, the project, people who use Docker, people who modify Docker, etc. This approach to contributions is inspired by the popular Linux Developer "Certificate of Origin". This approach makes it simple for new contributors to get started, and avoids bureaucracy in tracking contributions and contributors. To indicate accordance, each individual contribution to the Project is signed off by the developer, using his or her real name, email address, and github handle in the format below: Docker-DCO-1.0-Signed-off-by: Joe Smith --- CONTRIBUTING.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b4f2dc381..40174e770f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,6 +89,39 @@ name and email address match your git configuration. The AUTHORS file is regenerated occasionally from the git commit history, so a mismatch may result in your changes being overwritten. +### Sign your work + +The sign-off is a simple line at the end of the explanation for the +patch, which certifies that you wrote it or otherwise have the right to +pass it on as an open-source patch. The rules are pretty simple: if you +can certify the below: + +``` +Docker Developer Grant and Certificate of Origin 1.0 + +By making a contribution to the Docker Project ("Project"), I represent and warrant that: + +a. The contribution was created in whole or in part by me and I have the right to submit the contribution on my own behalf or on behalf of a third party who has authorized me to submit this contribution to the Project; or + + +b. The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right and authorization to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license) that I have identified in the contribution; or + +c. The contribution was provided directly to me by some other person who represented and warranted (a) or (b) and I have not modified it. + +d. I understand and agree that this Project and the contribution are publicly known and that a record of the contribution (including all personal information I submit with it, including my sign-off record) is maintained indefinitely and may be redistributed consistent with this Project or the open source license(s) involved. + +e. I hereby grant to the Project, dotCloud, Inc and its successors; and recipients of software distributed by the Project a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, modify, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute this contribution and such modifications and derivative works consistent with this Project, the open source license indicated in the previous work or other appropriate open source license specified by the Project and approved by the Open Source Initiative(OSI) at http://www.opensource.org. +``` + +then you just add a line saying + + Docker-DCO-1.0-Signed-off-by: Joe Smith @github_handle + +using your real name (sorry, no pseudonyms or anonymous contributions.) + +If you have any questions, please refer to the FAQ in the [docs](http://docs.docker.io) + + ### How can I become a maintainer? From b8cd2bc94d58e9ca1c6f9a41979b857cfa4b698f Mon Sep 17 00:00:00 2001 From: jaseg Date: Fri, 20 Dec 2013 14:02:10 +0100 Subject: [PATCH 02/41] Return meaningful error for meaningless Dockerfiles --- buildfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildfile.go b/buildfile.go index 7d87a17d3a..c819e9ae28 100644 --- a/buildfile.go +++ b/buildfile.go @@ -568,7 +568,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { } return b.image, nil } - return "", fmt.Errorf("An error occurred during the build\n") + return "", fmt.Errorf("No image was generated. This may be because the Dockerfile does not, like, do anything.\n") } func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeCache, rm bool, outOld io.Writer, sf *utils.StreamFormatter, auth *auth.AuthConfig) BuildFile { From 3264c1c5eb41d4504a813d955fdd9d3a0da8c2dd Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 23 Dec 2013 18:11:18 -0800 Subject: [PATCH 03/41] add REMOTE_TODO --- REMOTE_TODO.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 REMOTE_TODO.md diff --git a/REMOTE_TODO.md b/REMOTE_TODO.md new file mode 100644 index 0000000000..cd9460a392 --- /dev/null +++ b/REMOTE_TODO.md @@ -0,0 +1,46 @@ +``` +**GET** + send objects deprecate multi-stream +TODO "/events": getEvents, N +ok "/info": getInfo, 1 +ok "/version": getVersion, 1 +... "/images/json": getImagesJSON, N +TODO "/images/viz": getImagesViz, 0 yes +TODO "/images/search": getImagesSearch, N +TODO "/images/{name:.*}/get": getImagesGet, 0 +TODO "/images/{name:.*}/history": getImagesHistory, 1 +TODO "/images/{name:.*}/json": getImagesByName, 1 +TODO "/containers/ps": getContainersJSON, N +TODO "/containers/json": getContainersJSON, 1 +ok "/containers/{name:.*}/export": getContainersExport, 0 +TODO "/containers/{name:.*}/changes": getContainersChanges, 1 +TODO "/containers/{name:.*}/json": getContainersByName, 1 +TODO "/containers/{name:.*}/top": getContainersTop, N +TODO "/containers/{name:.*}/attach/ws": wsContainersAttach, 0 yes + +**POST** +TODO "/auth": postAuth, 0 yes +ok "/commit": postCommit, 0 +TODO "/build": postBuild, 0 yes +TODO "/images/create": postImagesCreate, N yes yes (pull) +TODO "/images/{name:.*}/insert": postImagesInsert, N yes yes +TODO "/images/load": postImagesLoad, 1 yes (stdin) +TODO "/images/{name:.*}/push": postImagesPush, N yes +ok "/images/{name:.*}/tag": postImagesTag, 0 +ok "/containers/create": postContainersCreate, 0 +ok "/containers/{name:.*}/kill": postContainersKill, 0 +TODO "/containers/{name:.*}/restart": postContainersRestart, 0 +ok "/containers/{name:.*}/start": postContainersStart, 0 +ok "/containers/{name:.*}/stop": postContainersStop, 0 +ok "/containers/{name:.*}/wait": postContainersWait, 0 +ok "/containers/{name:.*}/resize": postContainersResize, 0 +TODO "/containers/{name:.*}/attach": postContainersAttach, 0 yes +TODO "/containers/{name:.*}/copy": postContainersCopy, 0 yes + +**DELETE** +#3180 "/containers/{name:.*}": deleteContainers, 0 +TODO "/images/{name:.*}": deleteImages, N + +**OPTIONS** +ok "": optionsHandler, 0 +``` \ No newline at end of file From 8b82b0dfe74684b5ea30b7487475f7ca4abcc58e Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 28 Dec 2013 16:16:03 -0500 Subject: [PATCH 04/41] Make blank -H option default to the same as no -H was sent --- docs/sources/use/basics.rst | 5 +++++ utils/utils.go | 5 ++++- utils/utils_test.go | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 6b2c588817..a812d2b7b4 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -115,6 +115,11 @@ For example: * ``tcp://host:4243`` -> tcp connection on host:4243 * ``unix://path/to/socket`` -> unix socket located at ``path/to/socket`` +``-H``, when empty, will default to the same value as when no ``-H`` was passed in. + +``-H`` also accepts short form for TCP bindings: +``host[:port]`` or ``:port`` + .. code-block:: bash # Run docker in daemon mode diff --git a/utils/utils.go b/utils/utils.go index ffe90ac734..292fee1c55 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -803,7 +803,7 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s host string port int ) - + addr = strings.TrimSpace(addr) switch { case strings.HasPrefix(addr, "unix://"): proto = "unix" @@ -814,6 +814,9 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s case strings.HasPrefix(addr, "tcp://"): proto = "tcp" addr = strings.TrimPrefix(addr, "tcp://") + case addr == "": + proto = "unix" + addr = defaultUnix default: if strings.Contains(addr, "://") { return "", fmt.Errorf("Invalid bind address protocol: %s", addr) diff --git a/utils/utils_test.go b/utils/utils_test.go index b31938d9b5..bd3cf2274d 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -316,6 +316,9 @@ func TestParseHost(t *testing.T) { if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) } + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { + t.Errorf("empty argument -> expected unix:///var/run/docker.sock, got %s", addr) + } if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } From 83d631b6a455d3041b7cb31e1c7eb45a16715c43 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 29 Dec 2013 13:25:57 -0700 Subject: [PATCH 05/41] Fix "docker load" help by removing "SOURCE" prompt and mentioning STDIN Fixes #3383 --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 0627261d53..eeccf0a52e 100644 --- a/commands.go +++ b/commands.go @@ -2203,7 +2203,7 @@ func (cli *DockerCli) CmdSave(args ...string) error { } func (cli *DockerCli) CmdLoad(args ...string) error { - cmd := cli.Subcmd("load", "SOURCE", "Load an image from a tar archive") + cmd := cli.Subcmd("load", "", "Load an image from a tar archive on STDIN") if err := cmd.Parse(args); err != nil { return err } From c6dcee329dc9e720d70d9534c261fe9e9cf80512 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 30 Dec 2013 00:10:48 -0700 Subject: [PATCH 06/41] Add more TestBuild Dockerfiles, especially for making sure manual RUN echo '#!...' continues to work correctly --- integration/buildfile_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index dccee4a8d3..cc7eba1808 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -132,6 +132,23 @@ run [ "$(cat /e)" = "blah" ] [][2]string{{"/x", "hello"}, {"/", "blah"}}, }, + // Comments, shebangs, and executability, oh my! + { + ` +FROM {IMAGE} +# This is an ordinary comment. +RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh +RUN [ ! -x /hello.sh ] +RUN chmod +x /hello.sh +RUN [ -x /hello.sh ] +RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] +RUN [ "$(/hello.sh)" = "hello world" ] +`, + nil, + nil, + }, + + // Environment variable { ` from {IMAGE} @@ -142,6 +159,19 @@ run [ "$FOO" = "BAR" ] nil, }, + // Environment overwriting + { + ` +from {IMAGE} +env FOO BAR +run [ "$FOO" = "BAR" ] +env FOO BAZ +run [ "$FOO" = "BAZ" ] +`, + nil, + nil, + }, + { ` from {IMAGE} From 9dd7ae4074ecfceb62c3c3eae1811c5bd018c38c Mon Sep 17 00:00:00 2001 From: Roel Van Nyen Date: Thu, 2 Jan 2014 00:06:50 +0100 Subject: [PATCH 07/41] Cleanup: runtime.getFullName is deprecated by getFullName --- runtime.go | 9 ++------- server.go | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/runtime.go b/runtime.go index 7c7f722c5d..d4b096cfdd 100644 --- a/runtime.go +++ b/runtime.go @@ -582,11 +582,6 @@ func (runtime *Runtime) Commit(container *Container, repository, tag, comment, a return img, nil } -// FIXME: this is deprecated by the getFullName *function* -func (runtime *Runtime) getFullName(name string) (string, error) { - return getFullName(name) -} - func getFullName(name string) (string, error) { if name == "" { return "", fmt.Errorf("Container name cannot be empty") @@ -598,7 +593,7 @@ func getFullName(name string) (string, error) { } func (runtime *Runtime) GetByName(name string) (*Container, error) { - fullName, err := runtime.getFullName(name) + fullName, err := getFullName(name) if err != nil { return nil, err } @@ -614,7 +609,7 @@ func (runtime *Runtime) GetByName(name string) (*Container, error) { } func (runtime *Runtime) Children(name string) (map[string]*Container, error) { - name, err := runtime.getFullName(name) + name, err := getFullName(name) if err != nil { return nil, err } diff --git a/server.go b/server.go index 9f6842dbae..5ac2b0ee05 100644 --- a/server.go +++ b/server.go @@ -1435,7 +1435,7 @@ func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool) if container == nil { return fmt.Errorf("No such link: %s", name) } - name, err := srv.runtime.getFullName(name) + name, err := getFullName(name) if err != nil { return err } From 848f290012a06e84723cad5f9222697c367b0a61 Mon Sep 17 00:00:00 2001 From: "Carl X. Su" Date: Thu, 2 Jan 2014 12:56:29 +0800 Subject: [PATCH 08/41] Upgrade VBox Guest Additions (Fix #3425) --- Vagrantfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 54fc783c00..def85e5d72 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -70,7 +70,7 @@ SCRIPT # trigger dkms to build the virtualbox guest module install. $vbox_script = < Date: Thu, 2 Jan 2014 11:29:24 -0500 Subject: [PATCH 09/41] install instructions use unprefixed fedora image Signed-off-by: Lokesh Mandvekar --- docs/sources/installation/fedora.rst | 2 +- docs/sources/installation/rhel.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index de296b4df2..6dd2bf91d9 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -67,7 +67,7 @@ Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t mattdm/fedora /bin/bash + sudo docker run -i -t fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index b928b333f4..9036fb79ea 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -65,7 +65,7 @@ Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t mattdm/fedora /bin/bash + sudo docker run -i -t fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. From 589515c717616f3c2e0278ad787d21b4ab07b994 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 Jan 2014 17:51:42 +0100 Subject: [PATCH 10/41] Check standalone header when pinging a registry server. Standalone has to be true to use basic auth (in addition to previous requirements) --- registry/registry.go | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 6c9255aa46..17c2ccd887 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -25,11 +25,11 @@ var ( ErrLoginRequired = errors.New("Authentication is required.") ) -func pingRegistryEndpoint(endpoint string) error { +func pingRegistryEndpoint(endpoint string) (bool, error) { if endpoint == auth.IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) - return nil + return false, nil } httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds @@ -45,14 +45,26 @@ func pingRegistryEndpoint(endpoint string) error { client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { - return err + return false, err } defer resp.Body.Close() if resp.Header.Get("X-Docker-Registry-Version") == "" { - return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + return false, errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") } - return nil + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + utils.Debugf("Registry standalone header: '%s'", standalone) + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry + if standalone == "" { + return true, nil + // Accepted values are "true" (case-insensitive) and "1". + } else if strings.EqualFold(standalone, "true") || standalone == "1" { + return true, nil + } + // Otherwise, not standalone + return false, nil } func validateRepositoryName(repositoryName string) error { @@ -122,16 +134,16 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { // there is no path given. Expand with default path hostname = hostname + "/v1/" } - if err := pingRegistryEndpoint(hostname); err != nil { + if _, err := pingRegistryEndpoint(hostname); err != nil { return "", errors.New("Invalid Registry endpoint: " + err.Error()) } return hostname, nil } endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if err := pingRegistryEndpoint(endpoint); err != nil { + if _, err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if err = pingRegistryEndpoint(endpoint); err != nil { + if _, err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" return "", errors.New("Invalid Registry endpoint: " + err.Error()) } @@ -677,12 +689,18 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, return nil, err } - // If we're working with a private registry over HTTPS, send Basic Auth headers + // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) + standalone, err := pingRegistryEndpoint(indexEndpoint) + if err != nil { + return nil, err + } + if standalone { + utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } } r.reqFactory = factory From c561212b837735213f146c2fac291f84dbcbc8c3 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Sat, 21 Dec 2013 11:02:06 -0500 Subject: [PATCH 11/41] Extract cgroups utilities to own submodule. Use mountinfo rather than for cgroups parsing. Make helper method private and change name. Makes method naming more explicit rather than GetThisCgroup. Use upstream term subsystem rather than cgroupType. --- cgroups/MAINTAINERS | 1 + cgroups/cgroups.go | 101 ++++++++++++++++++++++++++++++++++++++++ cgroups/cgroups_test.go | 27 +++++++++++ mount/mount.go | 6 ++- mount/mountinfo.go | 49 +++++++++++++------ runtime.go | 5 +- server.go | 3 +- utils/utils.go | 78 ------------------------------- 8 files changed, 175 insertions(+), 95 deletions(-) create mode 100644 cgroups/MAINTAINERS create mode 100644 cgroups/cgroups.go create mode 100644 cgroups/cgroups_test.go diff --git a/cgroups/MAINTAINERS b/cgroups/MAINTAINERS new file mode 100644 index 0000000000..1e998f8ac1 --- /dev/null +++ b/cgroups/MAINTAINERS @@ -0,0 +1 @@ +Michael Crosby (@crosbymichael) diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go new file mode 100644 index 0000000000..30de8d4d1e --- /dev/null +++ b/cgroups/cgroups.go @@ -0,0 +1,101 @@ +package cgroups + +import ( + "bufio" + "fmt" + "github.com/dotcloud/docker/mount" + "io" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" +) + +// https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt + +func FindCgroupMountpoint(subsystem string) (string, error) { + mounts, err := mount.GetMounts() + if err != nil { + return "", err + } + + for _, mount := range mounts { + if mount.Fstype == "cgroup" { + for _, opt := range strings.Split(mount.VfsOpts, ",") { + if opt == subsystem { + return mount.Mountpoint, nil + } + } + } + } + + return "", fmt.Errorf("cgroup mountpoint not found for %s", subsystem) +} + +// Returns the relative path to the cgroup docker is running in. +func getThisCgroupDir(subsystem string) (string, error) { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + return parseCgroupFile(subsystem, f) +} + +func parseCgroupFile(subsystem string, r io.Reader) (string, error) { + s := bufio.NewScanner(r) + + for s.Scan() { + if err := s.Err(); err != nil { + return "", err + } + text := s.Text() + parts := strings.Split(text, ":") + if parts[1] == subsystem { + return parts[2], nil + } + } + return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", subsystem) +} + +// Returns a list of pids for the given container. +func GetPidsForContainer(id string) ([]int, error) { + pids := []int{} + + // memory is chosen randomly, any cgroup used by docker works + subsystem := "memory" + + cgroupRoot, err := FindCgroupMountpoint(subsystem) + if err != nil { + return pids, err + } + + cgroupDir, err := getThisCgroupDir(subsystem) + if err != nil { + return pids, err + } + + filename := filepath.Join(cgroupRoot, cgroupDir, id, "tasks") + if _, err := os.Stat(filename); os.IsNotExist(err) { + // With more recent lxc versions use, cgroup will be in lxc/ + filename = filepath.Join(cgroupRoot, cgroupDir, "lxc", id, "tasks") + } + + output, err := ioutil.ReadFile(filename) + if err != nil { + return pids, err + } + for _, p := range strings.Split(string(output), "\n") { + if len(p) == 0 { + continue + } + pid, err := strconv.Atoi(p) + if err != nil { + return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) + } + pids = append(pids, pid) + } + return pids, nil +} diff --git a/cgroups/cgroups_test.go b/cgroups/cgroups_test.go new file mode 100644 index 0000000000..537336a493 --- /dev/null +++ b/cgroups/cgroups_test.go @@ -0,0 +1,27 @@ +package cgroups + +import ( + "bytes" + "testing" +) + +const ( + cgroupsContents = `11:hugetlb:/ +10:perf_event:/ +9:blkio:/ +8:net_cls:/ +7:freezer:/ +6:devices:/ +5:memory:/ +4:cpuacct,cpu:/ +3:cpuset:/ +2:name=systemd:/user.slice/user-1000.slice/session-16.scope` +) + +func TestParseCgroups(t *testing.T) { + r := bytes.NewBuffer([]byte(cgroupsContents)) + _, err := parseCgroupFile("blkio", r) + if err != nil { + t.Fatal(err) + } +} diff --git a/mount/mount.go b/mount/mount.go index 253a4681ef..b087293a9d 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -4,6 +4,10 @@ import ( "time" ) +func GetMounts() ([]*MountInfo, error) { + return parseMountTable() +} + // Looks at /proc/self/mountinfo to determine of the specified // mountpoint has been mounted func Mounted(mountpoint string) (bool, error) { @@ -14,7 +18,7 @@ func Mounted(mountpoint string) (bool, error) { // Search the table for the mountpoint for _, e := range entries { - if e.mountpoint == mountpoint { + if e.Mountpoint == mountpoint { return true, nil } } diff --git a/mount/mountinfo.go b/mount/mountinfo.go index f4fd24b633..32996f05c8 100644 --- a/mount/mountinfo.go +++ b/mount/mountinfo.go @@ -5,22 +5,35 @@ import ( "fmt" "io" "os" + "strings" ) const ( - // We only parse upto the mountinfo because that is all we - // care about right now - mountinfoFormat = "%d %d %d:%d %s %s %s" + /* 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + + (1) mount ID: unique identifier of the mount (may be reused after umount) + (2) parent ID: ID of parent (or of self for the top of the mount tree) + (3) major:minor: value of st_dev for files on filesystem + (4) root: root of the mount within the filesystem + (5) mount point: mount point relative to the process's root + (6) mount options: per mount options + (7) optional fields: zero or more fields of the form "tag[:value]" + (8) separator: marks the end of the optional fields + (9) filesystem type: name of filesystem of the form "type[.subtype]" + (10) mount source: filesystem specific information or "none" + (11) super options: per super block options*/ + mountinfoFormat = "%d %d %d:%d %s %s %s " ) -// Represents one line from /proc/self/mountinfo -type procEntry struct { - id, parent, major, minor int - source, mountpoint, opts string +type MountInfo struct { + Id, Parent, Major, Minor int + Root, Mountpoint, Opts string + Fstype, Source, VfsOpts string } // Parse /proc/self/mountinfo because comparing Dev and ino does not work from bind mounts -func parseMountTable() ([]*procEntry, error) { +func parseMountTable() ([]*MountInfo, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err @@ -30,10 +43,10 @@ func parseMountTable() ([]*procEntry, error) { return parseInfoFile(f) } -func parseInfoFile(r io.Reader) ([]*procEntry, error) { +func parseInfoFile(r io.Reader) ([]*MountInfo, error) { var ( s = bufio.NewScanner(r) - out = []*procEntry{} + out = []*MountInfo{} ) for s.Scan() { @@ -42,14 +55,24 @@ func parseInfoFile(r io.Reader) ([]*procEntry, error) { } var ( - p = &procEntry{} + p = &MountInfo{} text = s.Text() ) + if _, err := fmt.Sscanf(text, mountinfoFormat, - &p.id, &p.parent, &p.major, &p.minor, - &p.source, &p.mountpoint, &p.opts); err != nil { + &p.Id, &p.Parent, &p.Major, &p.Minor, + &p.Root, &p.Mountpoint, &p.Opts); err != nil { return nil, fmt.Errorf("Scanning '%s' failed: %s", text, err) } + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + if len(postSeparatorFields) != 3 { + return nil, fmt.Errorf("Error did not find 3 fields post '-' in '%s'", text) + } + p.Fstype = postSeparatorFields[0] + p.Source = postSeparatorFields[1] + p.VfsOpts = postSeparatorFields[2] out = append(out, p) } return out, nil diff --git a/runtime.go b/runtime.go index 7c7f722c5d..b1d8988607 100644 --- a/runtime.go +++ b/runtime.go @@ -4,11 +4,12 @@ import ( "container/list" "fmt" "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/pkg/graphdb" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" _ "github.com/dotcloud/docker/graphdriver/devmapper" _ "github.com/dotcloud/docker/graphdriver/vfs" + "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -332,7 +333,7 @@ func (runtime *Runtime) restore() error { // FIXME: comment please! func (runtime *Runtime) UpdateCapabilities(quiet bool) { - if cgroupMemoryMountpoint, err := utils.FindCgroupMountpoint("memory"); err != nil { + if cgroupMemoryMountpoint, err := cgroups.FindCgroupMountpoint("memory"); err != nil { if !quiet { log.Printf("WARNING: %s\n", err) } diff --git a/server.go b/server.go index 9f6842dbae..6445c540b0 100644 --- a/server.go +++ b/server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" @@ -701,7 +702,7 @@ func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) { if !container.State.IsRunning() { return nil, fmt.Errorf("Container %s is not running", name) } - pids, err := utils.GetPidsForContainer(container.ID) + pids, err := cgroups.GetPidsForContainer(container.ID) if err != nil { return nil, err } diff --git a/utils/utils.go b/utils/utils.go index ffe90ac734..b0ba43d377 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -583,28 +583,6 @@ func CompareKernelVersion(a, b *KernelVersionInfo) int { return 0 } -func FindCgroupMountpoint(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return "", err - } - - // /proc/mounts has 6 fields per line, one mount per line, e.g. - // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, " ") - if len(parts) == 6 && parts[2] == "cgroup" { - for _, opt := range strings.Split(parts[3], ",") { - if opt == cgroupType { - return parts[1], nil - } - } - } - } - - return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - func GetKernelVersion() (*KernelVersionInfo, error) { var ( err error @@ -1157,59 +1135,3 @@ func CopyFile(src, dst string) (int64, error) { defer df.Close() return io.Copy(df, sf) } - -// Returns the relative path to the cgroup docker is running in. -func GetThisCgroup(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/self/cgroup") - if err != nil { - return "", err - } - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, ":") - // any type used by docker should work - if parts[1] == cgroupType { - return parts[2], nil - } - } - return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", cgroupType) -} - -// Returns a list of pids for the given container. -func GetPidsForContainer(id string) ([]int, error) { - pids := []int{} - - // memory is chosen randomly, any cgroup used by docker works - cgroupType := "memory" - - cgroupRoot, err := FindCgroupMountpoint(cgroupType) - if err != nil { - return pids, err - } - - cgroupThis, err := GetThisCgroup(cgroupType) - if err != nil { - return pids, err - } - - filename := filepath.Join(cgroupRoot, cgroupThis, id, "tasks") - if _, err := os.Stat(filename); os.IsNotExist(err) { - // With more recent lxc versions use, cgroup will be in lxc/ - filename = filepath.Join(cgroupRoot, cgroupThis, "lxc", id, "tasks") - } - - output, err := ioutil.ReadFile(filename) - if err != nil { - return pids, err - } - for _, p := range strings.Split(string(output), "\n") { - if len(p) == 0 { - continue - } - pid, err := strconv.Atoi(p) - if err != nil { - return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) - } - pids = append(pids, pid) - } - return pids, nil -} From f560b87a863916d6ea5b0bd863db654563ffa14d Mon Sep 17 00:00:00 2001 From: Tzu-Jung Lee Date: Thu, 2 Jan 2014 15:51:50 -0800 Subject: [PATCH 12/41] Fix #3149: port mapping of TCP + UDP Signed-off-by: Tzu-Jung Lee --- network.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/network.go b/network.go index 5ee5c4bee8..0d3a27ced7 100644 --- a/network.go +++ b/network.go @@ -248,12 +248,12 @@ type PortMapper struct { } func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { - mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() - if _, exists := mapper.tcpProxies[mapKey]; exists { - return fmt.Errorf("Port %s is already in use", mapKey) - } if _, isTCP := backendAddr.(*net.TCPAddr); isTCP { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() + if _, exists := mapper.tcpProxies[mapKey]; exists { + return fmt.Errorf("TCP Port %s is already in use", mapKey) + } backendPort := backendAddr.(*net.TCPAddr).Port backendIP := backendAddr.(*net.TCPAddr).IP if mapper.iptables != nil { @@ -270,6 +270,10 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { mapper.tcpProxies[mapKey] = proxy go proxy.Run() } else { + mapKey := (&net.UDPAddr{Port: port, IP: ip}).String() + if _, exists := mapper.udpProxies[mapKey]; exists { + return fmt.Errorf("UDP: Port %s is already in use", mapKey) + } backendPort := backendAddr.(*net.UDPAddr).Port backendIP := backendAddr.(*net.UDPAddr).IP if mapper.iptables != nil { @@ -290,8 +294,8 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { } func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error { - mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() if proto == "tcp" { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() backendAddr, ok := mapper.tcpMapping[mapKey] if !ok { return fmt.Errorf("Port tcp/%s is not mapped", mapKey) @@ -307,6 +311,7 @@ func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error { } delete(mapper.tcpMapping, mapKey) } else { + mapKey := (&net.UDPAddr{Port: port, IP: ip}).String() backendAddr, ok := mapper.udpMapping[mapKey] if !ok { return fmt.Errorf("Port udp/%s is not mapped", mapKey) From 42c7dc448f550c954aeba14cd20d1dff12845c73 Mon Sep 17 00:00:00 2001 From: Roel Van Nyen Date: Fri, 3 Jan 2014 15:47:22 +0100 Subject: [PATCH 13/41] Just pass the container in RegisterLinks(), no need to lookup by name again --- server.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/server.go b/server.go index a49a34d542..4b1934175f 100644 --- a/server.go +++ b/server.go @@ -1714,12 +1714,8 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) return nil, nil } -func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { +func (srv *Server) RegisterLinks(container *Container, hostConfig *HostConfig) error { runtime := srv.runtime - container := runtime.Get(name) - if container == nil { - return fmt.Errorf("No such container: %s", name) - } if hostConfig != nil && hostConfig.Links != nil { for _, l := range hostConfig.Links { @@ -1792,8 +1788,7 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { } } // Register any links from the host config before starting the container - // FIXME: we could just pass the container here, no need to lookup by name again. - if err := srv.RegisterLinks(name, &hostConfig); err != nil { + if err := srv.RegisterLinks(container, &hostConfig); err != nil { job.Error(err) return engine.StatusErr } From 32b9a429c5d36785a4a544d6e7236a95a4310983 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 3 Jan 2014 17:02:59 -0700 Subject: [PATCH 14/41] Change version to 0.7.3-dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f38fc5393f..c4181d00bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3 +0.7.3-dev From 3069bf946047e42b4aa468617caffb45829317b6 Mon Sep 17 00:00:00 2001 From: Mahesh Tiyyagura Date: Sat, 4 Jan 2014 11:24:48 +0530 Subject: [PATCH 15/41] Changelog date typo fix: 2013-01-02 -> 2014-01-02 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d8d756a8..0d986fcc9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.7.3 (2013-01-02) +## 0.7.3 (2014-01-02) #### Builder From 2a1181f404387f6d530a86a6196b40a6dd541c20 Mon Sep 17 00:00:00 2001 From: sudosurootdev Date: Sat, 4 Jan 2014 23:15:15 -0500 Subject: [PATCH 16/41] Spelling error. --- hack/make.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/make.sh b/hack/make.sh index 90683b6ead..fe0f9c175f 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -17,7 +17,7 @@ set -e # DO NOT CALL THIS SCRIPT DIRECTLY. # - The right way to call this script is to invoke "make" from # your checkout of the Docker repository. -# the Makefile will so a "docker build -t docker ." and then +# the Makefile will do a "docker build -t docker ." and then # "docker run hack/make.sh" in the resulting container image. # From 599009191a6d9aa157de10c296909829b93bc00e Mon Sep 17 00:00:00 2001 From: Christopher Rigor Date: Sun, 5 Jan 2014 17:07:16 +0800 Subject: [PATCH 17/41] Fix login message to say pull instead of push. --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 97bfda5194..aab0bb963a 100644 --- a/commands.go +++ b/commands.go @@ -1100,7 +1100,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { if err := pull(authConfig); err != nil { if err.Error() == registry.ErrLoginRequired.Error() { - fmt.Fprintln(cli.out, "\nPlease login prior to push:") + fmt.Fprintln(cli.out, "\nPlease login prior to pull:") if err := cli.CmdLogin(endpoint); err != nil { return err } From ea762c1a51a7dfbb08294c2b6fa105095dd193b3 Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Mon, 6 Jan 2014 09:29:33 -0800 Subject: [PATCH 18/41] Update instructions for mtu flag for Docker on GCE --- docs/sources/installation/google.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sources/installation/google.rst b/docs/sources/installation/google.rst index ff38e1e6e4..8d78a9f7bb 100644 --- a/docs/sources/installation/google.rst +++ b/docs/sources/installation/google.rst @@ -62,7 +62,8 @@ .. code-block:: bash - docker -d -mtu 1460 + docker-playground:~$ echo "DOCKER_OPTS="$DOCKER_OPTS -mtu 1460" | sudo tee -a /etc/defaults/docker + docker-playground:~$ sudo service docker restart 8. Start a new container: From d6ca05f7cb6fd75a77967049c22eac7104d94633 Mon Sep 17 00:00:00 2001 From: David Mcanulty Date: Mon, 6 Jan 2014 09:53:58 -0800 Subject: [PATCH 19/41] Simplified and streamlined Amazon Quickstart The amazon quickstart images are EBS backed and include cloudinit. I've personally tested these documentation changes as a new user to Docker. --- docs/sources/installation/amazon.rst | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/sources/installation/amazon.rst b/docs/sources/installation/amazon.rst index 273eceb05f..e8fdc2c1ca 100644 --- a/docs/sources/installation/amazon.rst +++ b/docs/sources/installation/amazon.rst @@ -26,18 +26,13 @@ Amazon QuickStart `_ menu on your AWS Console. - * When picking the source AMI for your instance type, select "Community - AMIs". + * Click the ``Select`` button for a 64Bit Ubuntu image. For example: Ubuntu Server 12.04.3 LTS - * Search for ``amd64 precise``. Pick one of the amd64 Ubuntu images. - - * If you choose a EBS enabled AMI, you'll also be able to launch a + * For testing you can use the default (possibly free) ``t1.micro`` instance (more info on `pricing - `_). ``t1.micro`` instances are - eligible for Amazon's Free Usage Tier. + `_). - * When you click select you'll be taken to the instance setup, and you're one - click away from having your Ubuntu VM up and running. + * Click the ``Next: Configure Instance Details`` button at the bottom right. 2. **Tell CloudInit to install Docker:** From 323c4b5211ecef4cffcbe0a97811ddd93c469164 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 27 Dec 2013 21:06:26 -0500 Subject: [PATCH 20/41] Use same error handling while unmarshalling args for CMD and ENTRYPOINT --- buildfile.go | 39 ++++++++++++++++------------------- integration/buildfile_test.go | 2 ++ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/buildfile.go b/buildfile.go index fb97f26dcc..ca35fc58e4 100644 --- a/buildfile.go +++ b/buildfile.go @@ -271,16 +271,30 @@ func (b *buildFile) CmdEnv(args string) error { return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } -func (b *buildFile) CmdCmd(args string) error { +func (b *buildFile) buildCmdFromJson(args string) []string { var cmd []string if err := json.Unmarshal([]byte(args), &cmd); err != nil { - utils.Debugf("Error unmarshalling: %s, setting cmd to /bin/sh -c", err) + utils.Debugf("Error unmarshalling: %s, setting to /bin/sh -c", err) cmd = []string{"/bin/sh", "-c", args} } - if err := b.commit("", cmd, fmt.Sprintf("CMD %v", cmd)); err != nil { + return cmd +} + +func (b *buildFile) CmdCmd(args string) error { + cmd := b.buildCmdFromJson(args) + b.config.Cmd = cmd + if err := b.commit("", b.config.Cmd, fmt.Sprintf("CMD %v", cmd)); err != nil { + return err + } + return nil +} + +func (b *buildFile) CmdEntrypoint(args string) error { + entrypoint := b.buildCmdFromJson(args) + b.config.Entrypoint = entrypoint + if err := b.commit("", b.config.Cmd, fmt.Sprintf("ENTRYPOINT %v", entrypoint)); err != nil { return err } - b.config.Cmd = cmd return nil } @@ -303,23 +317,6 @@ func (b *buildFile) CmdCopy(args string) error { return fmt.Errorf("COPY has been deprecated. Please use ADD instead") } -func (b *buildFile) CmdEntrypoint(args string) error { - if args == "" { - return fmt.Errorf("Entrypoint cannot be empty") - } - - var entrypoint []string - if err := json.Unmarshal([]byte(args), &entrypoint); err != nil { - b.config.Entrypoint = []string{"/bin/sh", "-c", args} - } else { - b.config.Entrypoint = entrypoint - } - if err := b.commit("", b.config.Cmd, fmt.Sprintf("ENTRYPOINT %s", args)); err != nil { - return err - } - return nil -} - func (b *buildFile) CmdWorkdir(workdir string) error { b.config.WorkingDir = workdir return b.commit("", b.config.Cmd, fmt.Sprintf("WORKDIR %v", workdir)) diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index dccee4a8d3..89878926ed 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -391,6 +391,8 @@ func TestBuildEntrypoint(t *testing.T) { } if img.Config.Entrypoint[0] != "/bin/echo" { + t.Log(img.Config.Entrypoint[0]) + t.Fail() } } From b308e33106832cd4452f473ff0a54c2031070e0a Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Mon, 6 Jan 2014 10:30:02 -0800 Subject: [PATCH 21/41] Tweak formatting for GCE instructions --- docs/sources/installation/google.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/google.rst b/docs/sources/installation/google.rst index 8d78a9f7bb..38aab019db 100644 --- a/docs/sources/installation/google.rst +++ b/docs/sources/installation/google.rst @@ -57,7 +57,7 @@ docker-playground:~$ curl get.docker.io | bash docker-playground:~$ sudo update-rc.d docker defaults -7. If running in zones: us-central1-a, europe-west1-1, and europe-west1-b, the docker daemon must be started with the `-mtu` flag. Without the flag, you may experience intermittent network pauses. +7. If running in zones: ``us-central1-a``, ``europe-west1-1``, and ``europe-west1-b``, the docker daemon must be started with the ``-mtu`` flag. Without the flag, you may experience intermittent network pauses. `See this issue `_ for more details. .. code-block:: bash From 5e8912e0e8537c526ee4f162bdade2656c2fa803 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 6 Jan 2014 21:04:44 +0100 Subject: [PATCH 22/41] Fixed registry unit tests --- registry/registry_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry/registry_test.go b/registry/registry_test.go index 69eb25b247..16bc431e55 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -23,10 +23,11 @@ func spawnTestRegistry(t *testing.T) *Registry { } func TestPingRegistryEndpoint(t *testing.T) { - err := pingRegistryEndpoint(makeURL("/v1/")) + standalone, err := pingRegistryEndpoint(makeURL("/v1/")) if err != nil { t.Fatal(err) } + assertEqual(t, standalone, true, "Expected standalone to be true (default)") } func TestGetRemoteHistory(t *testing.T) { From 36ab1836f973c62ff543456f1613dec8813d00df Mon Sep 17 00:00:00 2001 From: Nick Stinemates Date: Mon, 6 Jan 2014 12:26:53 -0800 Subject: [PATCH 23/41] Fix dotCloud to Docker, Inc --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a2d39b071..359106b49f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -126,7 +126,7 @@ c. The contribution was provided directly to me by some other person who represe d. I understand and agree that this Project and the contribution are publicly known and that a record of the contribution (including all personal information I submit with it, including my sign-off record) is maintained indefinitely and may be redistributed consistent with this Project or the open source license(s) involved. -e. I hereby grant to the Project, dotCloud, Inc and its successors; and recipients of software distributed by the Project a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, modify, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute this contribution and such modifications and derivative works consistent with this Project, the open source license indicated in the previous work or other appropriate open source license specified by the Project and approved by the Open Source Initiative(OSI) at http://www.opensource.org. +e. I hereby grant to the Project, Docker, Inc and its successors; and recipients of software distributed by the Project a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, modify, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute this contribution and such modifications and derivative works consistent with this Project, the open source license indicated in the previous work or other appropriate open source license specified by the Project and approved by the Open Source Initiative(OSI) at http://www.opensource.org. ``` then you just add a line saying From 7799ae27ca1dd85761f0595346a0dda15bbeda6c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 23 Dec 2013 23:36:58 +0000 Subject: [PATCH 24/41] Move utility package 'iptables' to pkg/iptables --- integration/iptables_test.go | 2 +- links.go | 2 +- network.go | 2 +- network_test.go | 3 ++- {iptables => pkg/iptables}/MAINTAINERS | 0 {iptables => pkg/iptables}/iptables.go | 0 6 files changed, 5 insertions(+), 4 deletions(-) rename {iptables => pkg/iptables}/MAINTAINERS (100%) rename {iptables => pkg/iptables}/iptables.go (100%) diff --git a/integration/iptables_test.go b/integration/iptables_test.go index 060d0fe074..1dd4194350 100644 --- a/integration/iptables_test.go +++ b/integration/iptables_test.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "os" "testing" ) diff --git a/links.go b/links.go index 2fe255b4c5..55834b92d2 100644 --- a/links.go +++ b/links.go @@ -2,7 +2,7 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "path" "strings" ) diff --git a/network.go b/network.go index 5ee5c4bee8..f02275d37a 100644 --- a/network.go +++ b/network.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "github.com/dotcloud/docker/pkg/netlink" "github.com/dotcloud/docker/proxy" "github.com/dotcloud/docker/utils" diff --git a/network_test.go b/network_test.go index 184b497938..69fcba01a2 100644 --- a/network_test.go +++ b/network_test.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "github.com/dotcloud/docker/proxy" "net" "testing" @@ -340,6 +340,7 @@ func NewStubProxy(frontendAddr, backendAddr net.Addr) (proxy.Proxy, error) { } func TestPortMapper(t *testing.T) { + // FIXME: is this iptables chain still used anywhere? var chain *iptables.Chain mapper := &PortMapper{ tcpMapping: make(map[string]*net.TCPAddr), diff --git a/iptables/MAINTAINERS b/pkg/iptables/MAINTAINERS similarity index 100% rename from iptables/MAINTAINERS rename to pkg/iptables/MAINTAINERS diff --git a/iptables/iptables.go b/pkg/iptables/iptables.go similarity index 100% rename from iptables/iptables.go rename to pkg/iptables/iptables.go From 1e551c7cc5badc043b1d6a651c3f036c996ef538 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Mon, 6 Jan 2014 16:18:00 -0800 Subject: [PATCH 25/41] [docs] Fix for wrong version warning on master instead of latest. --- docs/theme/docker/static/js/docs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/theme/docker/static/js/docs.js b/docs/theme/docker/static/js/docs.js index 4151ea5a80..03401909fa 100755 --- a/docs/theme/docker/static/js/docs.js +++ b/docs/theme/docker/static/js/docs.js @@ -92,7 +92,7 @@ $(function(){ $('.version-flyer ul').html('
  • Local
    • '); } - if (doc_version == "master") { + if (doc_version == "latest") { $('.version-flyer .version-note').hide(); } From 6fe3da9924fc22bd02eba0803abb42b10fc28baf Mon Sep 17 00:00:00 2001 From: Nick Stinemates Date: Mon, 6 Jan 2014 17:21:14 -0800 Subject: [PATCH 26/41] Remove the @github handle as it generates emails. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 359106b49f..783c8273f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,7 +131,7 @@ e. I hereby grant to the Project, Docker, Inc and its successors; and recipient then you just add a line saying - Docker-DCO-1.0-Signed-off-by: Joe Smith @github_handle + Docker-DCO-1.0-Signed-off-by: Joe Smith (github: github_handle) using your real name (sorry, no pseudonyms or anonymous contributions.) From e1c861cf331b416c45a5e79102de3b44b13bf1a0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 6 Jan 2014 17:37:30 -0800 Subject: [PATCH 27/41] Update REMOTE_TODO.md --- REMOTE_TODO.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/REMOTE_TODO.md b/REMOTE_TODO.md index cd9460a392..0b688c3924 100644 --- a/REMOTE_TODO.md +++ b/REMOTE_TODO.md @@ -29,7 +29,7 @@ TODO "/images/{name:.*}/push": postImagesPush, N ok "/images/{name:.*}/tag": postImagesTag, 0 ok "/containers/create": postContainersCreate, 0 ok "/containers/{name:.*}/kill": postContainersKill, 0 -TODO "/containers/{name:.*}/restart": postContainersRestart, 0 +#3476 "/containers/{name:.*}/restart": postContainersRestart, 0 ok "/containers/{name:.*}/start": postContainersStart, 0 ok "/containers/{name:.*}/stop": postContainersStop, 0 ok "/containers/{name:.*}/wait": postContainersWait, 0 @@ -43,4 +43,4 @@ TODO "/images/{name:.*}": deleteImages, N **OPTIONS** ok "": optionsHandler, 0 -``` \ No newline at end of file +``` From 444a087ac2ca34776b2b3e4433aa35c27cdfd57a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 Jan 2014 17:37:49 -0800 Subject: [PATCH 28/41] Make CopyFileWithTar use a pipe instead of a buffer. Save more than 1Gb of ram for a 500Mb file. --- archive/archive.go | 55 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/archive/archive.go b/archive/archive.go index f8fcf0b163..1b8076543f 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -3,8 +3,8 @@ package archive import ( "archive/tar" "bytes" - "compress/gzip" "compress/bzip2" + "compress/gzip" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -299,7 +299,7 @@ func CopyWithTar(src, dst string) error { // // If `dst` ends with a trailing slash '/', the final destination path // will be `dst/base(src)`. -func CopyFileWithTar(src, dst string) error { +func CopyFileWithTar(src, dst string) (err error) { utils.Debugf("CopyFileWithTar(%s, %s)", src, dst) srcSt, err := os.Stat(src) if err != nil { @@ -316,25 +316,38 @@ func CopyFileWithTar(src, dst string) error { if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { return err } - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - hdr, err := tar.FileInfoHeader(srcSt, "") - if err != nil { - return err - } - hdr.Name = filepath.Base(dst) - if err := tw.WriteHeader(hdr); err != nil { - return err - } - srcF, err := os.Open(src) - if err != nil { - return err - } - if _, err := io.Copy(tw, srcF); err != nil { - return err - } - tw.Close() - return Untar(buf, filepath.Dir(dst), nil) + + r, w := io.Pipe() + errC := utils.Go(func() error { + defer w.Close() + + srcF, err := os.Open(src) + if err != nil { + return err + } + defer srcF.Close() + + tw := tar.NewWriter(w) + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = filepath.Base(dst) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + tw.Close() + return nil + }) + defer func() { + if er := <-errC; err != nil { + err = er + } + }() + return Untar(r, filepath.Dir(dst), nil) } // CmdStream executes a command, and returns its stdout as a stream. From 8f3b8f383519452001cc05bf6505e9c301ebe954 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 6 Jan 2014 22:14:35 -0700 Subject: [PATCH 29/41] Run 'gofmt -s -w' --- engine/http.go | 4 ++-- reflink_copy_darwin.go | 2 +- reflink_copy_linux.go | 2 +- utils.go | 1 - utils/utils.go | 2 +- utils/utils_test.go | 6 +++--- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/engine/http.go b/engine/http.go index 6391b3ff5a..b115912e2c 100644 --- a/engine/http.go +++ b/engine/http.go @@ -1,8 +1,8 @@ package engine import ( - "path" "net/http" + "path" ) // ServeHTTP executes a job as specified by the http request `r`, and sends the @@ -22,7 +22,7 @@ func (eng *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request) { jobArgs = []string{} } w.Header().Set("Job-Name", jobName) - for _, arg := range(jobArgs) { + for _, arg := range jobArgs { w.Header().Add("Job-Args", arg) } job := eng.Job(jobName, jobArgs...) diff --git a/reflink_copy_darwin.go b/reflink_copy_darwin.go index 3f3147db0f..4f0ea8c4fd 100644 --- a/reflink_copy_darwin.go +++ b/reflink_copy_darwin.go @@ -1,8 +1,8 @@ package docker import ( - "os" "io" + "os" ) func CopyFile(dstFile, srcFile *os.File) error { diff --git a/reflink_copy_linux.go b/reflink_copy_linux.go index 8aae3abf3d..83c7f75413 100644 --- a/reflink_copy_linux.go +++ b/reflink_copy_linux.go @@ -25,8 +25,8 @@ btrfs_reflink(int fd_out, int fd_in) import "C" import ( - "os" "io" + "os" "syscall" ) diff --git a/utils.go b/utils.go index 5bcc678184..3eb1eac045 100644 --- a/utils.go +++ b/utils.go @@ -322,7 +322,6 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error { return nil } - // Links come in the format of // name:alias func parseLink(rawLink string) (map[string]string, error) { diff --git a/utils/utils.go b/utils/utils.go index 25c8012c54..d5dbf35f0a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -781,7 +781,7 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s host string port int ) - addr = strings.TrimSpace(addr) + addr = strings.TrimSpace(addr) switch { case strings.HasPrefix(addr, "unix://"): proto = "unix" diff --git a/utils/utils_test.go b/utils/utils_test.go index bd3cf2274d..1f23755d11 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -316,9 +316,9 @@ func TestParseHost(t *testing.T) { if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { - t.Errorf("empty argument -> expected unix:///var/run/docker.sock, got %s", addr) - } + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { + t.Errorf("empty argument -> expected unix:///var/run/docker.sock, got %s", addr) + } if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } From 2b93f1822322e957e7652db2645462ef2ec256bd Mon Sep 17 00:00:00 2001 From: lukaspustina Date: Tue, 7 Jan 2014 15:20:38 +0100 Subject: [PATCH 30/41] Missing pipe in line 85 --- docs/sources/use/networking.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/use/networking.rst b/docs/sources/use/networking.rst index 0e81440e03..4e75fbc20d 100644 --- a/docs/sources/use/networking.rst +++ b/docs/sources/use/networking.rst @@ -82,7 +82,7 @@ In this scenario: $ sudo ifconfig bridge0 192.168.227.1 netmask 255.255.255.0 # Edit your Docker startup file - $ echo "DOCKER_OPTS=\"-b=bridge0\"" /etc/default/docker + $ echo "DOCKER_OPTS=\"-b=bridge0\"" >> /etc/default/docker # Start Docker $ sudo service docker start From 561d1db074edd4e7e7d8b02401abd3fc11b9d51c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 4 Dec 2013 13:47:37 -0800 Subject: [PATCH 31/41] Add Travis CI configuration to validate DCO and gofmt After each push, Travis CI will trigger, and check two things: - make sure that each commit in the push has the Docker certificate of origin - make sure that all .go files changed by this sequence of commits are correctly formatted in the most recent commit Note: there is one edge case; if you do a git force push, we cannot figure out the actual commits in the force push, and we will just run the checks as if upstream master were the base. Pull requests will always be tested correctly, though. Docker-DCO-1.0-Signed-off-by: Andrew Page (github: tianon) --- .travis.yml | 22 ++++++++++++++++++++++ MAINTAINERS | 1 + hack/travis/dco.py | 41 +++++++++++++++++++++++++++++++++++++++++ hack/travis/env.py | 21 +++++++++++++++++++++ hack/travis/gofmt.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+) create mode 100644 .travis.yml create mode 100755 hack/travis/dco.py create mode 100644 hack/travis/env.py create mode 100755 hack/travis/gofmt.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..b40a16d8fb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +# Note: right now we don't use go-specific features of travis. +# Later we might automate "go test" etc. (or do it inside a docker container...?) + +language: go + +go: 1.2 + +# Disable the normal go build. +install: true + +before_script: + - env | sort + - sudo apt-get update -qq + - sudo apt-get install -qq python-yaml + - git remote add upstream git://github.com/dotcloud/docker.git + - git fetch upstream +refs/heads/master:refs/remotes/upstream/master + +script: + - hack/travis/dco.py + - hack/travis/gofmt.py + +# vim:set sw=2 ts=2: diff --git a/MAINTAINERS b/MAINTAINERS index ef3aeda493..4a6c0ec22c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2,6 +2,7 @@ Solomon Hykes (@shykes) Guillaume Charmes (@creack) Victor Vieux (@vieux) Michael Crosby (@crosbymichael) +.travis.yml: Tianon Gravi (@tianon) api.go: Victor Vieux (@vieux) Dockerfile: Tianon Gravi (@tianon) Makefile: Tianon Gravi (@tianon) diff --git a/hack/travis/dco.py b/hack/travis/dco.py new file mode 100755 index 0000000000..bca84bfe75 --- /dev/null +++ b/hack/travis/dco.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +import re +import subprocess +import yaml + +from env import commit_range + +commit_format = '-%n hash: %h%n author: %aN <%aE>%n message: |%n%w(0,2,2)%B' + +gitlog = subprocess.check_output([ + 'git', 'log', '--reverse', + '--format=format:'+commit_format, + '..'.join(commit_range), '--', +]) + +commits = yaml.load(gitlog) +if not commits: + exit(0) # what? how can we have no commits? + +DCO = 'Docker-DCO-1.0-Signed-off-by:' + +p = re.compile(r'^{0} ([^<]+) <([^<>@]+@[^<>]+)> \(github: (\S+)\)$'.format(re.escape(DCO)), re.MULTILINE|re.UNICODE) + +failed_commits = 0 + +for commit in commits: + m = p.search(commit['message']) + if not m: + print 'Commit {1} does not have a properly formatted "{0}" marker.'.format(DCO, commit['hash']) + failed_commits += 1 + continue # print ALL the commits that don't have a proper DCO + + (name, email, github) = m.groups() + + # TODO verify that "github" is the person who actually made this commit via the GitHub API + +if failed_commits > 0: + exit(failed_commits) + +print 'All commits have a valid "{0}" marker.'.format(DCO) +exit(0) diff --git a/hack/travis/env.py b/hack/travis/env.py new file mode 100644 index 0000000000..86d90f1567 --- /dev/null +++ b/hack/travis/env.py @@ -0,0 +1,21 @@ +import os +import subprocess + +if 'TRAVIS' not in os.environ: + print 'TRAVIS is not defined; this should run in TRAVIS. Sorry.' + exit(127) + +if os.environ['TRAVIS_PULL_REQUEST'] != 'false': + commit_range = [os.environ['TRAVIS_BRANCH'], 'FETCH_HEAD'] +else: + try: + subprocess.check_call([ + 'git', 'log', '-1', '--format=format:', + os.environ['TRAVIS_COMMIT_RANGE'], '--', + ]) + commit_range = os.environ['TRAVIS_COMMIT_RANGE'].split('...') + if len(commit_range) == 1: # if it didn't split, it must have been separated by '..' instead + commit_range = commit_range[0].split('..') + except subprocess.CalledProcessError: + print 'TRAVIS_COMMIT_RANGE is invalid. This seems to be a force push. We will just assume it must be against upstream master and compare all commits in between.' + commit_range = ['upstream/master', 'HEAD'] diff --git a/hack/travis/gofmt.py b/hack/travis/gofmt.py new file mode 100755 index 0000000000..1bb062e2f6 --- /dev/null +++ b/hack/travis/gofmt.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import subprocess + +from env import commit_range + +files = subprocess.check_output([ + 'git', 'diff', '--diff-filter=ACMR', + '--name-only', '...'.join(commit_range), '--', +]) + +exit_status = 0 + +for filename in files.split('\n'): + if filename.endswith('.go'): + try: + out = subprocess.check_output(['gofmt', '-s', '-l', filename]) + if out != '': + print out, + exit_status = 1 + except subprocess.CalledProcessError: + exit_status = 1 + +if exit_status != 0: + print 'Reformat the files listed above with "gofmt -s -w" and try again.' + exit(exit_status) + +print 'All files pass gofmt.' +exit(0) From 97ef8a067c77cb8d15e6e336264c6eb21b25a0fb Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 7 Jan 2014 11:46:07 -0800 Subject: [PATCH 32/41] Fixed #3481 - Added Ubuntu Saucy to installation Docker-DCO-1.0-Signed-off-by: James Turnbull jamtur01 --- docs/sources/installation/ubuntulinux.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 00400a94bc..e4432c6710 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -17,7 +17,7 @@ Ubuntu Docker is supported on the following versions of Ubuntu: - :ref:`ubuntu_precise` -- :ref:`ubuntu_raring` +- :ref:`ubuntu_raring_saucy` Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ @@ -108,10 +108,12 @@ Type ``exit`` to exit **Done!**, now continue with the :ref:`hello_world` example. -.. _ubuntu_raring: +.. _ubuntu_raring_saucy: -Ubuntu Raring 13.04 (64 bit) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Ubuntu Raring 13.04 and Saucy 13.10 (64 bit) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These instructions cover both Ubuntu Raring 13.04 and Saucy 13.10. Dependencies ------------ @@ -169,7 +171,6 @@ Type ``exit`` to exit **Done!**, now continue with the :ref:`hello_world` example. - .. _ufw: Docker and UFW From 7c4e5fbd4608bb4b1509552fbe9d3c0bcd7c347a Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 7 Jan 2014 12:51:51 -0700 Subject: [PATCH 33/41] Fix Travis PR testing to not overwrite the "master" that Travis creates with our "upstream master", and to also ignore commits with no content (like a Merge commit - there's no need to sign off on literally nothing) Docker-DCO-1.0-Signed-off-by: Andrew Page (github: tianon) --- .travis.yml | 3 ++- hack/travis/dco.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b40a16d8fb..f6c83997aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ before_script: - sudo apt-get update -qq - sudo apt-get install -qq python-yaml - git remote add upstream git://github.com/dotcloud/docker.git - - git fetch upstream +refs/heads/master:refs/remotes/upstream/master + - git fetch --append --no-tags upstream refs/heads/master:refs/remotes/upstream/master +# sometimes we have upstream master already as origin/master (PRs), but other times we don't, so let's just make sure we have a completely unambiguous way to specify "upstream master" from here out script: - hack/travis/dco.py diff --git a/hack/travis/dco.py b/hack/travis/dco.py index bca84bfe75..6a875863a1 100755 --- a/hack/travis/dco.py +++ b/hack/travis/dco.py @@ -24,6 +24,14 @@ p = re.compile(r'^{0} ([^<]+) <([^<>@]+@[^<>]+)> \(github: (\S+)\)$'.format(re.e failed_commits = 0 for commit in commits: + commit['stat'] = subprocess.check_output([ + 'git', 'log', '--format=format:', '--max-count=1', + '--name-status', commit['hash'], '--', + ]) + if commit['stat'] == '': + print 'Commit {0} has no actual changed content, skipping.'.format(commit['hash']) + continue + m = p.search(commit['message']) if not m: print 'Commit {1} does not have a properly formatted "{0}" marker.'.format(DCO, commit['hash']) From f2ea539467e34ef06455a3e0561bac1f5a482822 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 7 Jan 2014 12:51:17 -0800 Subject: [PATCH 34/41] Update REMOTE_TODO.md --- REMOTE_TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REMOTE_TODO.md b/REMOTE_TODO.md index 0b688c3924..8caddc68ba 100644 --- a/REMOTE_TODO.md +++ b/REMOTE_TODO.md @@ -7,7 +7,7 @@ ok "/version": getVersion, 1 ... "/images/json": getImagesJSON, N TODO "/images/viz": getImagesViz, 0 yes TODO "/images/search": getImagesSearch, N -TODO "/images/{name:.*}/get": getImagesGet, 0 +#3490 "/images/{name:.*}/get": getImagesGet, 0 TODO "/images/{name:.*}/history": getImagesHistory, 1 TODO "/images/{name:.*}/json": getImagesByName, 1 TODO "/containers/ps": getContainersJSON, N From fa14a1b9835a46bc9c48c1385b514a0f8d0b4eac Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 7 Jan 2014 11:34:19 -0800 Subject: [PATCH 35/41] Only get rootfs when we need to calculate the image size Docker-DCO-1.0-Signed-off-by: Michael Crosby (github: crosbymichael) --- graph.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/graph.go b/graph.go index 058ef7e93d..176626d60a 100644 --- a/graph.go +++ b/graph.go @@ -87,17 +87,17 @@ func (graph *Graph) Get(name string) (*Image, error) { if err != nil { return nil, err } - // Check that the filesystem layer exists - rootfs, err := graph.driver.Get(img.ID) - if err != nil { - return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err) - } if img.ID != id { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) } img.graph = graph if img.Size < 0 { + rootfs, err := graph.driver.Get(img.ID) + if err != nil { + return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err) + } + var size int64 if img.Parent == "" { if size, err = utils.TreeSize(rootfs); err != nil { From 3779291e9b2f939aa98b91b35552bd9b76c9f39a Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 7 Jan 2014 15:25:38 -0700 Subject: [PATCH 36/41] Fix Travis handling of a fully-numeric commit hash being YAML-converted to a number instead of a string Docker-DCO-1.0-Signed-off-by: Andrew Page (github: tianon) --- hack/travis/dco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/travis/dco.py b/hack/travis/dco.py index 6a875863a1..cab26fedd7 100755 --- a/hack/travis/dco.py +++ b/hack/travis/dco.py @@ -5,7 +5,7 @@ import yaml from env import commit_range -commit_format = '-%n hash: %h%n author: %aN <%aE>%n message: |%n%w(0,2,2)%B' +commit_format = '-%n hash: "%h"%n author: %aN <%aE>%n message: |%n%w(0,2,2)%B' gitlog = subprocess.check_output([ 'git', 'log', '--reverse', From 811341423bd9d3d65640cb4aa04d2840817182ca Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 Jan 2014 16:25:48 -0800 Subject: [PATCH 37/41] Fix ADD caching issue with . prefixed path Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 7 ++++ integration/buildfile_test.go | 71 ++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/buildfile.go b/buildfile.go index 09b1f80739..8e92702ae0 100644 --- a/buildfile.go +++ b/buildfile.go @@ -412,6 +412,13 @@ func (b *buildFile) CmdAdd(args string) error { } else if fi.IsDir() { var subfiles []string for file, sum := range sums { + // Has tarsum stips the '.' and './', we put it back for comparaison. + if len(file) == 0 { + file = "./" + } + if file[0] != '.' && file[0] != '/' { + file = "./" + file + } if strings.HasPrefix(file, origPath) { subfiles = append(subfiles, sum) } diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 89878926ed..5f3991636d 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -427,7 +427,7 @@ func TestBuildEntrypointRunCleanup(t *testing.T) { } } -func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bool) { +func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bool) (imageId string) { eng := NewTestEngine(t) defer nuke(mkRuntimeFromEngine(eng, t)) @@ -436,20 +436,36 @@ func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bo t.Fatal(err) } - imageId := img.ID + imageId = img.ID - img = nil img, err = buildImage(template, t, eng, expectHit) if err != nil { t.Fatal(err) } - hit := imageId == img.ID - if hit != expectHit { - t.Logf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", - hit, expectHit, imageId, img.ID) - t.Fail() + if hit := imageId == img.ID; hit != expectHit { + t.Fatalf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", hit, expectHit, imageId, img.ID) } + return +} + +func checkCacheBehaviorFromEngime(t *testing.T, template testContextTemplate, expectHit bool, eng *engine.Engine) (imageId string) { + img, err := buildImage(template, t, eng, true) + if err != nil { + t.Fatal(err) + } + + imageId = img.ID + + img, err = buildImage(template, t, eng, expectHit) + if err != nil { + t.Fatal(err) + } + + if hit := imageId == img.ID; hit != expectHit { + t.Fatalf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", hit, expectHit, imageId, img.ID) + } + return } func TestBuildImageWithCache(t *testing.T) { @@ -476,11 +492,46 @@ func TestBuildADDLocalFileWithCache(t *testing.T) { maintainer dockerio run echo "first" add foo /usr/lib/bla/bar + run [ "$(cat /usr/lib/bla/bar)" = "hello" ] run echo "second" + add . /src/ + run [ "$(cat /src/foo)" = "hello" ] `, - [][2]string{{"foo", "hello"}}, + [][2]string{ + {"foo", "hello"}, + }, nil} - checkCacheBehavior(t, template, true) + eng := NewTestEngine(t) + defer nuke(mkRuntimeFromEngine(eng, t)) + + id1 := checkCacheBehaviorFromEngime(t, template, true, eng) + template.files = append(template.files, [2]string{"bar", "hello2"}) + id2 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id1 == id2 { + t.Fatal("The cache should have been invalided but hasn't.") + } + id3 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id2 != id3 { + t.Fatal("The cache should have been used but hasn't.") + } + template.files[1][1] = "hello3" + id4 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id3 == id4 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.dockerfile += ` + add ./bar /src2/ + run ls /src2/bar + ` + id5 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id4 == id5 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.files[1][1] = "hello4" + id6 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id5 == id6 { + t.Fatal("The cache should have been invalided but hasn't.") + } } func TestBuildADDLocalFileWithoutCache(t *testing.T) { From ef7e000a13eed5ebf761ab4ca83fa8f727170aa5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 7 Jan 2014 16:02:41 -0800 Subject: [PATCH 38/41] Make vfs error more explicit Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- graphdriver/vfs/driver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/graphdriver/vfs/driver.go b/graphdriver/vfs/driver.go index fab9d06f83..12230f463a 100644 --- a/graphdriver/vfs/driver.go +++ b/graphdriver/vfs/driver.go @@ -36,9 +36,8 @@ func (d *Driver) Cleanup() error { } func copyDir(src, dst string) error { - cmd := exec.Command("cp", "-aT", "--reflink=auto", src, dst) - if err := cmd.Run(); err != nil { - return err + if output, err := exec.Command("cp", "-aT", "--reflink=auto", src, dst).CombinedOutput(); err != nil { + return fmt.Errorf("Error VFS copying directory: %s (%s)", err, output) } return nil } From f3103e5c9157aed2a5ab0f4c70e328cd0aa69b59 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 7 Jan 2014 16:53:55 -0800 Subject: [PATCH 39/41] Fix issue with file caching + prevent wrong cache hit Docker-DCO-1.0-Signed-off-by: Guillaume J. Charmes (github: creack) --- buildfile.go | 19 +++++++++++-------- integration/buildfile_test.go | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/buildfile.go b/buildfile.go index 8e92702ae0..ef4b95c064 100644 --- a/buildfile.go +++ b/buildfile.go @@ -407,18 +407,20 @@ func (b *buildFile) CmdAdd(args string) error { hash string sums = b.context.GetSums() ) + + // Has tarsum strips the '.' and './', we put it back for comparaison. + for file, sum := range sums { + if len(file) == 0 || file[0] != '.' && file[0] != '/' { + delete(sums, file) + sums["./"+file] = sum + } + } + if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { return err } else if fi.IsDir() { var subfiles []string for file, sum := range sums { - // Has tarsum stips the '.' and './', we put it back for comparaison. - if len(file) == 0 { - file = "./" - } - if file[0] != '.' && file[0] != '/' { - file = "./" + file - } if strings.HasPrefix(file, origPath) { subfiles = append(subfiles, sum) } @@ -435,7 +437,8 @@ func (b *buildFile) CmdAdd(args string) error { if err != nil { return err } - if hit { + // If we do not have a hash, never use the cache + if hit && hash != "" { return nil } } diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 5f3991636d..efbdc54951 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -532,6 +532,21 @@ func TestBuildADDLocalFileWithCache(t *testing.T) { if id5 == id6 { t.Fatal("The cache should have been invalided but hasn't.") } + + template.dockerfile += ` + add bar /src2/bar2 + add /bar /src2/bar3 + run ls /src2/bar2 /src2/bar3 + ` + id7 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id6 == id7 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.files[1][1] = "hello5" + id8 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id7 == id8 { + t.Fatal("The cache should have been invalided but hasn't.") + } } func TestBuildADDLocalFileWithoutCache(t *testing.T) { From d003cfea25c276904dbe1e972c0cf71d5c25e689 Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 8 Jan 2014 00:54:42 +0200 Subject: [PATCH 40/41] Revert "Add -S option to tar for efficient sparse file handling" This reverts commit 733bf5d3ddbfb6dba7c2c0996c4af47a765e4593. This is needed to fix "no such file" errors `docker build` errors for devicemapper. This fixes issue #3449. Docker-DCO-1.0-Signed-off-by: Cristian Staretu (github: unclejack) --- AUTHORS | 1 - archive/archive.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index f9e9ecdace..1ff9b8db02 100644 --- a/AUTHORS +++ b/AUTHORS @@ -48,7 +48,6 @@ Daniel YC Lin Darren Coxall David Calavera David Sissitka -Dinesh Subhraveti Deni Bertovic Dominik Honnef Don Spaulding diff --git a/archive/archive.go b/archive/archive.go index 1b8076543f..4dd5f006ef 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -149,7 +149,7 @@ func escapeName(name string) string { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. func TarFilter(path string, options *TarOptions) (io.Reader, error) { - args := []string{"tar", "-S", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"} + args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"} if options.Includes == nil { options.Includes = []string{"."} } @@ -228,7 +228,7 @@ func Untar(archive io.Reader, path string, options *TarOptions) error { compression := DetectCompression(buf) utils.Debugf("Archive compression detected: %s", compression.Extension()) - args := []string{"-S", "--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()} + args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()} if options != nil { for _, exclude := range options.Excludes { From 010d74ec2f3dfefea5cafa602bce5cdeac3a0c19 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 7 Jan 2014 18:00:49 -0800 Subject: [PATCH 41/41] Bump to v0.7.4 Docker-DCO-1.0-Signed-off-by: Michael Crosby (github: crosbymichael) --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d986fcc9b..eccf52ed00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## 0.7.4 (2014-01-07) + +#### Builder + +- Fix ADD caching issue with . prefixed path +- Fix docker build on devicemapper by reverting sparse file tar option +- Fix issue with file caching and prevent wrong cache hit +* Use same error handling while unmarshalling CMD and ENTRYPOINT + +#### Documentation + +* Simplify and streamline Amazon Quickstart +* Install instructions use unprefixed fedora image +* Update instructions for mtu flag for Docker on GCE ++ Add Ubuntu Saucy to installation +- Fix for wrong version warning on master instead of latest + +#### Runtime + +- Only get the image's rootfs when we need to calculate the image size +- Correctly handle unmapping UDP ports +* Make CopyFileWithTar use a pipe instead of a buffer to save memory on docker build +- Fix login message to say pull instead of push +- Fix "docker load" help by removing "SOURCE" prompt and mentioning STDIN +* Make blank -H option default to the same as no -H was sent +* Extract cgroups utilities to own submodule + +#### Other + ++ Add Travis CI configuration to validate DCO and gofmt requirements ++ Add Developer Certificate of Origin Text +* Upgrade VBox Guest Additions +* Check standalone header when pinging a registry server + ## 0.7.3 (2014-01-02) #### Builder diff --git a/VERSION b/VERSION index c4181d00bb..0a1ffad4b4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3-dev +0.7.4