From 64450ae3f89b8f9b5288224c5a7d109a166cf22a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 3 Jul 2013 17:11:00 +0000 Subject: [PATCH 01/54] add last version --- commands.go | 9 +++++++++ utils/utils.go | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/commands.go b/commands.go index 6e1e5e88c2..e275bdd671 100644 --- a/commands.go +++ b/commands.go @@ -407,6 +407,15 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if out.GoVersion != "" { fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } + + release := utils.GetReleaseVersion() + if release != "" { + fmt.Fprintf(cli.out, "Last stable version: %s", release) + if VERSION != release || out.Version != release { + fmt.Fprintf(cli.out, ", please update docker") + } + fmt.Fprintf(cli.out, "\n") + } return nil } diff --git a/utils/utils.go b/utils/utils.go index 2f2a52867e..b4a41ea420 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -687,3 +687,25 @@ func ParseHost(host string, port int, addr string) string { } return fmt.Sprintf("tcp://%s:%d", host, port) } + +func GetReleaseVersion() string { + type githubTag struct { + Name string `json:"name"` + } + + resp, err := http.Get("https://api.github.com/repos/dotcloud/docker/tags") + if err != nil { + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + var tags []githubTag + err = json.Unmarshal(body, &tags) + if err != nil || len(tags) == 0 { + return "" + } + return strings.TrimPrefix(tags[0].Name, "v") +} From 24dd50490a027f01ea086eb90663d53348fa770e Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:36:05 +0100 Subject: [PATCH 02/54] docker.upstart: avoid spawning a `sh` process start script / end script create an intermediate sh process. --- packaging/ubuntu/docker.upstart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packaging/ubuntu/docker.upstart b/packaging/ubuntu/docker.upstart index 2bd5565ee7..1d35d7a493 100644 --- a/packaging/ubuntu/docker.upstart +++ b/packaging/ubuntu/docker.upstart @@ -4,6 +4,4 @@ start on runlevel [2345] stop on starting rc RUNLEVEL=[016] respawn -script - /usr/bin/docker -d -end script +exec /usr/bin/docker -d From 0900d3b7a6e94bfa42e3d4ac6dc6f5542f65a9b0 Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:41:19 +0100 Subject: [PATCH 03/54] docker.upstart: use the same start/stop events as sshd Is probably more solid --- packaging/ubuntu/docker.upstart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/ubuntu/docker.upstart b/packaging/ubuntu/docker.upstart index 1d35d7a493..f4d2fbe922 100644 --- a/packaging/ubuntu/docker.upstart +++ b/packaging/ubuntu/docker.upstart @@ -1,7 +1,8 @@ description "Run docker" -start on runlevel [2345] -stop on starting rc RUNLEVEL=[016] +start on filesystem or runlevel [2345] +stop on runlevel [!2345] + respawn exec /usr/bin/docker -d From 0afed3eded0950b5899729714fb11d6a7322301e Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Jul 2013 20:56:41 +0000 Subject: [PATCH 04/54] handle -dev --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 5f3913f9c2..7b70faa198 100644 --- a/commands.go +++ b/commands.go @@ -436,7 +436,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { release := utils.GetReleaseVersion() if release != "" { fmt.Fprintf(cli.out, "Last stable version: %s", release) - if VERSION != release || out.Version != release { + if strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release { fmt.Fprintf(cli.out, ", please update docker") } fmt.Fprintf(cli.out, "\n") From cfec1c3e1b88ceeca73144198df7a210ed3dc421 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:06:32 +0000 Subject: [PATCH 05/54] add ps args to docker top --- api.go | 6 +++++- api_params.go | 6 ++---- api_test.go | 27 ++++++++++++++++++--------- commands.go | 17 +++++++++++------ server.go | 29 ++++++++++++++--------------- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/api.go b/api.go index b6ab7badfa..d3b84df5f9 100644 --- a/api.go +++ b/api.go @@ -255,8 +255,12 @@ func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *ht if vars == nil { return fmt.Errorf("Missing parameter") } + if err := parseForm(r); err != nil { + return err + } name := vars["name"] - procsStr, err := srv.ContainerTop(name) + ps_args := r.Form.Get("ps_args") + procsStr, err := srv.ContainerTop(name, ps_args) if err != nil { return err } diff --git a/api_params.go b/api_params.go index b371ca314f..2296ee792e 100644 --- a/api_params.go +++ b/api_params.go @@ -27,10 +27,8 @@ type APIInfo struct { } type APITop struct { - PID string - Tty string - Time string - Cmd string + Titles []string + Processes [][]string } type APIRmi struct { diff --git a/api_test.go b/api_test.go index 17ada96eab..9b7f08d1db 100644 --- a/api_test.go +++ b/api_test.go @@ -444,24 +444,33 @@ func TestGetContainersTop(t *testing.T) { } r := httptest.NewRecorder() - if err := getContainersTop(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil { + req, err := http.NewRequest("GET", "/"+container.ID+"/top?ps_args=u", bytes.NewReader([]byte{})) + if err != nil { t.Fatal(err) } - procs := []APITop{} + if err := getContainersTop(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + procs := APITop{} if err := json.Unmarshal(r.Body.Bytes(), &procs); err != nil { t.Fatal(err) } - if len(procs) != 2 { - t.Fatalf("Expected 2 processes, found %d.", len(procs)) + if len(procs.Titles) != 11 { + t.Fatalf("Expected 11 titles, found %d.", len(procs.Titles)) + } + if procs.Titles[0] != "USER" || procs.Titles[10] != "COMMAND" { + t.Fatalf("Expected Titles[0] to be USER and Titles[10] to be COMMAND, found %s and %s.", procs.Titles[0], procs.Titles[10]) } - if procs[0].Cmd != "sh" && procs[0].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[0].Cmd) + if len(procs.Processes) != 2 { + t.Fatalf("Expected 2 processes, found %d.", len(procs.Processes)) } - - if procs[1].Cmd != "sh" && procs[1].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[1].Cmd) + if procs.Processes[0][10] != "/bin/sh" && procs.Processes[0][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[0][10]) + } + if procs.Processes[1][10] != "/bin/sh" && procs.Processes[1][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[1][10]) } } diff --git a/commands.go b/commands.go index 936b23fea2..b25e928efa 100644 --- a/commands.go +++ b/commands.go @@ -585,23 +585,28 @@ func (cli *DockerCli) CmdTop(args ...string) error { if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() != 1 { + if cmd.NArg() == 0 { cmd.Usage() return nil } - body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top", nil) + val := url.Values{} + if cmd.NArg() > 1 { + val.Set("ps_args", strings.Join(cmd.Args()[1:], " ")) + } + + body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top?"+val.Encode(), nil) if err != nil { return err } - var procs []APITop + procs := APITop{} err = json.Unmarshal(body, &procs) if err != nil { return err } w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) - fmt.Fprintln(w, "PID\tTTY\tTIME\tCMD") - for _, proc := range procs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", proc.PID, proc.Tty, proc.Time, proc.Cmd) + fmt.Fprintln(w, strings.Join(procs.Titles, "\t")) + for _, proc := range procs.Processes { + fmt.Fprintln(w, strings.Join(proc, "\t")) } w.Flush() return nil diff --git a/server.go b/server.go index 954bbb208f..ae5f605267 100644 --- a/server.go +++ b/server.go @@ -249,35 +249,34 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { } -func (srv *Server) ContainerTop(name string) ([]APITop, error) { +func (srv *Server) ContainerTop(name, ps_args string) (*APITop, error) { if container := srv.runtime.Get(name); container != nil { - output, err := exec.Command("lxc-ps", "--name", container.ID).CombinedOutput() + output, err := exec.Command("lxc-ps", "--name", container.ID, "--", ps_args).CombinedOutput() if err != nil { return nil, fmt.Errorf("Error trying to use lxc-ps: %s (%s)", err, output) } - var procs []APITop + procs := APITop{} for i, line := range strings.Split(string(output), "\n") { - if i == 0 || len(line) == 0 { + if len(line) == 0 { continue } - proc := APITop{} + words := []string{} scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) if !scanner.Scan() { return nil, fmt.Errorf("Error trying to use lxc-ps") } // no scanner.Text because we skip container id - scanner.Scan() - proc.PID = scanner.Text() - scanner.Scan() - proc.Tty = scanner.Text() - scanner.Scan() - proc.Time = scanner.Text() - scanner.Scan() - proc.Cmd = scanner.Text() - procs = append(procs, proc) + for scanner.Scan() { + words = append(words, scanner.Text()) + } + if i == 0 { + procs.Titles = words + } else { + procs.Processes = append(procs.Processes, words) + } } - return procs, nil + return &procs, nil } return nil, fmt.Errorf("No such container: %s", name) From eb4a0271fbb7318a7d66911ccd388c048db0c2fa Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:34:55 +0000 Subject: [PATCH 06/54] bump api version to 1.4 --- api.go | 5 +- docs/sources/api/docker_remote_api.rst | 17 +- docs/sources/api/docker_remote_api_v1.4.rst | 1093 +++++++++++++++++++ 3 files changed, 1112 insertions(+), 3 deletions(-) create mode 100644 docs/sources/api/docker_remote_api_v1.4.rst diff --git a/api.go b/api.go index d3b84df5f9..a3dd52108e 100644 --- a/api.go +++ b/api.go @@ -17,7 +17,7 @@ import ( "strings" ) -const APIVERSION = 1.3 +const APIVERSION = 1.4 const DEFAULTHTTPHOST string = "127.0.0.1" const DEFAULTHTTPPORT int = 4243 @@ -252,6 +252,9 @@ func getContainersChanges(srv *Server, version float64, w http.ResponseWriter, r } func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if version < 1.4 { + return fmt.Errorf("top was improved a lot since 1.3, Please upgrade your docker client.") + } if vars == nil { return fmt.Errorf("Missing parameter") } diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 183347c23b..a08fb46940 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -19,8 +19,8 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 -Calling /images//insert is the same as calling /v1.3/images//insert +The current verson of the API is 1.4 +Calling /images//insert is the same as calling /v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert :doc:`docker_remote_api_v1.3` @@ -31,6 +31,18 @@ What's new Listing processes (/top): +- You can now use ps args with docker top, like `docker top aux` + +:doc:`docker_remote_api_v1.3` +***************************** + +docker v0.5.0 51f6c4a_ + +What's new +---------- + +Listing processes (/top): + - List the processes inside a container @@ -109,6 +121,7 @@ Initial version .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4 .. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168 +.. _51f6c4a: https://github.com/dotcloud/docker/commit/51f6c4a7372450d164c61e0054daf0223ddbd909 ================================== Docker Remote API Client Libraries diff --git a/docs/sources/api/docker_remote_api_v1.4.rst b/docs/sources/api/docker_remote_api_v1.4.rst new file mode 100644 index 0000000000..c42adb286f --- /dev/null +++ b/docs/sources/api/docker_remote_api_v1.4.rst @@ -0,0 +1,1093 @@ +:title: Remote API v1.3 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +====================== +Docker Remote API v1.3 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker deamon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"] + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: tag to be applied to the resulting image in case of success + :query q: suppress verbose build output + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From e701dce33978a0206627a02b258da6a0269c4b60 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Tue, 23 Jul 2013 13:05:06 -0700 Subject: [PATCH 07/54] Docs: Fixed navigaton links to about page and community page Website: Removed the website sources from the repo. The website sources are now hosted on github.com/dotcloud/www.docker.io/ --- docs/theme/MAINTAINERS | 2 +- docs/theme/docker/layout.html | 8 +- docs/website/MAINTAINERS | 1 - docs/website/dotcloud.yml | 2 - docs/website/gettingstarted/index.html | 220 --------------- docs/website/index.html | 359 ------------------------- docs/website/nginx.conf | 6 - docs/website/static | 1 - 8 files changed, 5 insertions(+), 594 deletions(-) delete mode 100644 docs/website/MAINTAINERS delete mode 100644 docs/website/dotcloud.yml delete mode 100644 docs/website/gettingstarted/index.html delete mode 100644 docs/website/index.html delete mode 100644 docs/website/nginx.conf delete mode 120000 docs/website/static diff --git a/docs/theme/MAINTAINERS b/docs/theme/MAINTAINERS index 6df367c073..606a1dd746 100644 --- a/docs/theme/MAINTAINERS +++ b/docs/theme/MAINTAINERS @@ -1 +1 @@ -Thatcher Penskens +Thatcher Peskens diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 198cd5d7d8..0b22f22fab 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -68,12 +68,12 @@
diff --git a/docs/website/MAINTAINERS b/docs/website/MAINTAINERS deleted file mode 100644 index 6df367c073..0000000000 --- a/docs/website/MAINTAINERS +++ /dev/null @@ -1 +0,0 @@ -Thatcher Penskens diff --git a/docs/website/dotcloud.yml b/docs/website/dotcloud.yml deleted file mode 100644 index 5a8f50f9e9..0000000000 --- a/docs/website/dotcloud.yml +++ /dev/null @@ -1,2 +0,0 @@ -www: - type: static \ No newline at end of file diff --git a/docs/website/gettingstarted/index.html b/docs/website/gettingstarted/index.html deleted file mode 100644 index de0cc3512d..0000000000 --- a/docs/website/gettingstarted/index.html +++ /dev/null @@ -1,220 +0,0 @@ - - - - - - - - - - Docker - the Linux container runtime - - - - - - - - - - - - - - - - - - - - -
-
-
- -
- -
- -
- -
-
-
-
- - -
-
- -
- - - -

GETTING STARTED

-
- -
- -
- -
-
- Docker is still under heavy development. It should not yet be used in production. Check the repo for recent progress. -
-
-
-
-

- - Installing on Ubuntu

- -

Requirements

-
    -
  • Ubuntu 12.04 (LTS) (64-bit)
    • -
  • or Ubuntu 12.10 (quantal) (64-bit)
    • -
  • The 3.8 Linux Kernel
    • -
-
    -
  1. -

    Install dependencies

    - The linux-image-extra package is only needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - 
    sudo apt-get install linux-image-extra-`uname -r`
    - - -
    2. -
  2. -

    Install Docker

    -

    Add the Ubuntu PPA (Personal Package Archive) sources to your apt sources list, update and install.

    -

    This may import a new GPG key (key 63561DC6: public key "Launchpad PPA for dotcloud team" imported).

    -
    - 
    sudo apt-get install software-properties-common
    - 
    sudo add-apt-repository ppa:dotcloud/lxc-docker
    - 
    sudo apt-get update
    - 
    sudo apt-get install lxc-docker
    -
    - - -
    3. - -
  3. -

    Run!

    - -
    - 
    docker run -i -t ubuntu /bin/bash
    -
    -
    4. - Continue with the Hello world example.
    - Or check more detailed installation instructions -
-
- -
-

Contributing to Docker

- -

Want to hack on Docker? Awesome! We have some instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete.

-
- -
-
-
-

Quick install on other operating systems

-

For other operating systems we recommend and provide a streamlined install with virtualbox, - vagrant and an Ubuntu virtual machine.

- - - -
- -
-

Questions? Want to get in touch?

-

There are several ways to get in touch:

-

Join the discussion on IRC. We can be found in the #docker channel on chat.freenode.net

-

Discussions happen on our google group: docker-club at googlegroups.com

-

All our development and decisions are made out in the open on Github github.com/dotcloud/docker

-

Get help on using Docker by asking on Stackoverflow

-

And of course, tweet your tweets to twitter.com/getdocker

-
- - -
-
- Fill out my online form. -
- -
- -
-
-
- - -
- -
- - - - - - - - - - - diff --git a/docs/website/index.html b/docs/website/index.html deleted file mode 100644 index f6f4efbccb..0000000000 --- a/docs/website/index.html +++ /dev/null @@ -1,359 +0,0 @@ - - - - - - - - - - - Docker - the Linux container engine - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - - -
- -
-
-
-
- - -
-
- -
-
- -
-
- docker letters - -

The Linux container engine

-
- -
- -
- Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers which are independent of hardware, language, framework, packaging system and hosting provider. -
- -
- - - - -
- -
-
- -
-
- -
-
-
-
-
- -
-
- -
-
-

Heterogeneous payloads

-

Any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all.

-

Any server

-

Docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments.

-

Isolation

-

Docker isolates processes from each other and from the underlying host, using lightweight containers.

-

Repeatability

-

Because each container is isolated in its own filesystem, they behave the same regardless of where, when, and alongside what they run.

-
-
-
-
- we're hiring -
-
-

Do you think it is cool to hack on docker? Join us!

-
    -
  • Work on open source
    • -
  • Program in Go
    • -
- read more -
-
- -
-
-
-
-

New! Docker Index

- On the Docker Index you can find and explore pre-made container images. It allows you to share your images and download them. - -

- -
- DOCKER index -
- -   - - -
-
-
- Fill out my online form. -
- -
-
-
- -
- -
- -
-
-
- - Mitchell Hashimoto ‏@mitchellh: Docker launched today. It is incredible. They’re also working RIGHT NOW on a Vagrant provider. LXC is COMING!! -
-
-
-
- - Adam Jacob ‏@adamhjk: Docker is clearly the right idea. @solomonstre absolutely killed it. Containerized app deployment is the future, I think. -
-
-
-
-
-
- - Matt Townsend ‏@mtownsend: I have a serious code crush on docker.io - it's Lego for PaaS. Motherfucking awesome Lego. -
-
-
-
- - Rob Harrop ‏@robertharrop: Impressed by @getdocker - it's all kinds of magic. Serious rethink of AWS architecture happening @skillsmatter. -
-
-
-
-
-
- - John Willis @botchagalupe: IMHO docker is to paas what chef was to Iaas 4 years ago -
-
-
-
- - John Feminella ‏@superninjarobot: So, @getdocker is pure excellence. If you've ever wished for arbitrary, PaaS-agnostic, lxc/aufs Linux containers, this is your jam! -
-
-
-
-
-
- - David Romulan ‏@destructuring: I haven't had this much fun since AWS -
-
-
-
- - Ricardo Gladwell ‏@rgladwell: wow @getdocker is either amazing or totally stupid -
-
- -
-
- -
-
-
- -
- -

Notable features

- -
    -
  • Filesystem isolation: each process container runs in a completely separate root filesystem.
    • -
  • Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups.
    • -
  • Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own.
    • -
  • Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremely fast, memory-cheap and disk-cheap.
    • -
  • Logging: the standard streams (stdout/stderr/stdin) of each process container is collected and logged for real-time or batch retrieval.
    • -
  • Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required.
    • -
  • Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell.
    • -
- -

Under the hood

- -

Under the hood, Docker is built on the following components:

- -
    -
  • The cgroup and namespacing capabilities of the Linux kernel;
    • -
  • AUFS, a powerful union filesystem with copy-on-write capabilities;
    • -
  • The Go programming language;
    • -
  • lxc, a set of convenience scripts to simplify the creation of linux containers.
    • -
- -

Who started it

-

- Docker is an open-source implementation of the deployment engine which powers dotCloud, a popular Platform-as-a-Service.

- -

It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands - of applications and databases. -

- -
-
- -
- - -
-

Twitter

- - -
- -
-
- -
- - -
- -
- - - - - - - - - - - - diff --git a/docs/website/nginx.conf b/docs/website/nginx.conf deleted file mode 100644 index 97ffd2c0e5..0000000000 --- a/docs/website/nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ - -# rule to redirect original links created when hosted on github pages -rewrite ^/documentation/(.*).html http://docs.docker.io/en/latest/$1/ permanent; - -# rewrite the stuff which was on the current page -rewrite ^/gettingstarted.html$ /gettingstarted/ permanent; diff --git a/docs/website/static b/docs/website/static deleted file mode 120000 index 95bc97aa10..0000000000 --- a/docs/website/static +++ /dev/null @@ -1 +0,0 @@ -../theme/docker/static \ No newline at end of file From f385f1860bdebe2434bb122bd5ac8fec85687970 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:18:34 +0000 Subject: [PATCH 08/54] ensure mount in commit --- builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder.go b/builder.go index ab07233cb8..420370b1e6 100644 --- a/builder.go +++ b/builder.go @@ -124,6 +124,10 @@ func (builder *Builder) Create(config *Config) (*Container, error) { func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. + if err := container.EnsureMounted(); err != nil { + return nil, err + } + rwTar, err := container.ExportRw() if err != nil { return nil, err From 48833c7b0784bc7055f29dd3448df3af93c462ec Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:20:56 +0000 Subject: [PATCH 09/54] add regression test + go fmt --- server_test.go | 21 +++++++++++++++++++++ utils/utils.go | 7 +++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/server_test.go b/server_test.go index 8612b3fcea..f67b8fb57c 100644 --- a/server_test.go +++ b/server_test.go @@ -90,6 +90,27 @@ func TestCreateRm(t *testing.T) { } +func TestCommit(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) + if err != nil { + t.Fatal(err) + } + + id, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { + t.Fatal(err) + } +} + func TestCreateStartRestartStopStartKillRm(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) diff --git a/utils/utils.go b/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 3425c1b84c3f58ac5bb2feb91c4901b61561c58c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 23 Jul 2013 11:37:13 -0700 Subject: [PATCH 10/54] Make sure the cookie is used in all registry queries --- registry/registry.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index e6f4f592e2..adef1c7baa 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -109,7 +109,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } - return c.Do(req) + res, err := c.Do(req) + if err != nil { + return nil, err + } + if len(res.Cookies()) > 0 { + c.Jar.SetCookies(req.URL, res.Cookies()) + } + return res, err } // Set the user agent field in the header based on the versions provided @@ -135,7 +142,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) @@ -182,7 +189,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -210,7 +217,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -231,7 +238,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -326,7 +333,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } @@ -341,9 +348,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) - } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { From d86898b0142f2f9a834aa0c727b10d62ef647262 Mon Sep 17 00:00:00 2001 From: Fareed Dudhia Date: Tue, 9 Jul 2013 07:01:45 +0000 Subject: [PATCH 11/54] Fixes 1136; Reopened from 1175 with latest changes. --- buildfile.go | 69 ++++++++++++++++++++++++++++++++++++++++------- buildfile_test.go | 42 +++++++++++++++++++++++++++-- utils/utils.go | 7 +++-- 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/buildfile.go b/buildfile.go index 75ebdd7a7c..c5171aaa91 100644 --- a/buildfile.go +++ b/buildfile.go @@ -11,6 +11,7 @@ import ( "os" "path" "reflect" + "regexp" "strings" ) @@ -67,6 +68,9 @@ func (b *buildFile) CmdFrom(name string) error { } b.image = image.ID b.config = &Config{} + if b.config.Env == nil || len(b.config.Env) == 0 { + b.config.Env = append(b.config.Env, "HOME=/", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + } return nil } @@ -112,6 +116,40 @@ func (b *buildFile) CmdRun(args string) error { return nil } +func (b *buildFile) FindEnvKey(key string) int { + for k, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + if key == envParts[0] { + return k + } + } + return -1 +} + +func (b *buildFile) ReplaceEnvMatches(value string) (string, error) { + exp, err := regexp.Compile("(\\\\\\\\+|[^\\\\]|\\b|\\A)\\$({?)([[:alnum:]_]+)(}?)") + if err != nil { + return value, err + } + matches := exp.FindAllString(value, -1) + for _, match := range matches { + match = match[strings.Index(match, "$"):] + matchKey := strings.Trim(match, "${}") + + for _, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + envKey := envParts[0] + envValue := envParts[1] + + if envKey == matchKey { + value = strings.Replace(value, match, envValue, -1) + break + } + } + } + return value, nil +} + func (b *buildFile) CmdEnv(args string) error { tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -120,14 +158,19 @@ func (b *buildFile) CmdEnv(args string) error { key := strings.Trim(tmp[0], " \t") value := strings.Trim(tmp[1], " \t") - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } + envKey := b.FindEnvKey(key) + replacedValue, err := b.ReplaceEnvMatches(value) + if err != nil { + return err } - b.config.Env = append(b.config.Env, key+"="+value) - return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s=%s", key, value)) + replacedVar := fmt.Sprintf("%s=%s", key, replacedValue) + + if envKey >= 0 { + b.config.Env[envKey] = replacedVar + return nil + } + b.config.Env = append(b.config.Env, replacedVar) + return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } func (b *buildFile) CmdCmd(args string) error { @@ -260,8 +303,16 @@ func (b *buildFile) CmdAdd(args string) error { if len(tmp) != 2 { return fmt.Errorf("Invalid ADD format") } - orig := strings.Trim(tmp[0], " \t") - dest := strings.Trim(tmp[1], " \t") + + orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t")) + if err != nil { + return err + } + + dest, err := b.ReplaceEnvMatches(strings.Trim(tmp[1], " \t")) + if err != nil { + return err + } cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} diff --git a/buildfile_test.go b/buildfile_test.go index b7eca52336..78e53b8419 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -129,6 +129,38 @@ CMD Hello world nil, nil, }, + + { + ` +from {IMAGE} +env FOO /foo/baz +env BAR /bar +env BAZ $BAR +env FOOPATH $PATH:$FOO +run [ "$BAR" = "$BAZ" ] +run [ "$FOOPATH" = "$PATH:/foo/baz" ] +`, + nil, + nil, + }, + + { + ` +from {IMAGE} +env FOO /bar +env TEST testdir +env BAZ /foobar +add testfile $BAZ/ +add $TEST $FOO +run [ "$(cat /foobar/testfile)" = "test1" ] +run [ "$(cat /bar/withfile)" = "test2" ] +`, + [][2]string{ + {"testfile", "test1"}, + {"testdir/withfile", "test2"}, + }, + nil, + }, } // FIXME: test building with 2 successive overlapping ADD commands @@ -242,8 +274,14 @@ func TestBuildEnv(t *testing.T) { env port 4243 `, nil, nil}, t) - - if img.Config.Env[0] != "port=4243" { + hasEnv := false + for _, envVar := range img.Config.Env { + if envVar == "port=4243" { + hasEnv = true + break + } + } + if !hasEnv { t.Fail() } } diff --git a/utils/utils.go b/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 4ebe2cf348915415c34503aca7a5663177e0002f Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 01:10:42 -0700 Subject: [PATCH 12/54] Change reserve-compatibility to reverse-compatibility --- container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container.go b/container.go index ec4abffc1b..d0b6ca4ce2 100644 --- a/container.go +++ b/container.go @@ -52,7 +52,7 @@ type Container struct { waitLock chan struct{} Volumes map[string]string - // Store rw/ro in a separate structure to preserve reserve-compatibility on-disk. + // Store rw/ro in a separate structure to preserve reverse-compatibility on-disk. // Easier than migrating older container configs :) VolumesRW map[string]bool } From e608296bc62ceeaf41ebf2bc80b21c0a1883d4f0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 09:19:26 +0000 Subject: [PATCH 13/54] fix wrong untag when using rmi via id --- server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server.go b/server.go index ce1fc8eaf8..fa8d8a0262 100644 --- a/server.go +++ b/server.go @@ -995,6 +995,9 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo + if len(strings.Split(repoAndTag, ":")) > 1 { + tag = strings.Split(repoAndTag, ":")[1] + } } else if repoName != parsedRepo { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict From 513a5674831fd952fc8e7b8fbdbc4939393ecb5b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:04:46 +0000 Subject: [PATCH 14/54] fix docs --- docs/sources/api/docker_remote_api.rst | 6 +++--- docs/sources/api/docker_remote_api_v1.4.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index d07c7634b9..0a1ff21cf1 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -26,15 +26,15 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 +The current verson of the API is 1.4 Calling /images//insert is the same as calling -/v1.3/images//insert +/v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert -:doc:`docker_remote_api_v1.3` +:doc:`docker_remote_api_v1.4` ***************************** What's new diff --git a/docs/sources/api/docker_remote_api_v1.4.rst b/docs/sources/api/docker_remote_api_v1.4.rst index c42adb286f..6ee0b35fa2 100644 --- a/docs/sources/api/docker_remote_api_v1.4.rst +++ b/docs/sources/api/docker_remote_api_v1.4.rst @@ -1,9 +1,9 @@ -:title: Remote API v1.3 +:title: Remote API v1.4 :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation ====================== -Docker Remote API v1.3 +Docker Remote API v1.4 ====================== .. contents:: Table of Contents From e592f1b298c778d0b9adfd6751f5fe1843a7001d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:01:41 +0000 Subject: [PATCH 15/54] add regression test --- server.go | 2 +- server_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index fa8d8a0262..56f738a5cd 100644 --- a/server.go +++ b/server.go @@ -995,7 +995,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo - if len(strings.Split(repoAndTag, ":")) > 1 { + if len(srv.runtime.repositories.ByID()[img.ID]) == 1 && len(strings.Split(repoAndTag, ":")) > 1 { tag = strings.Split(repoAndTag, ":")[1] } } else if repoName != parsedRepo { diff --git a/server_test.go b/server_test.go index 8612b3fcea..b6b21cc75f 100644 --- a/server_test.go +++ b/server_test.go @@ -2,6 +2,7 @@ package docker import ( "github.com/dotcloud/docker/utils" + "strings" "testing" "time" ) @@ -203,3 +204,88 @@ func TestLogEvent(t *testing.T) { }) } + +func TestRmi(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + srv := &Server{runtime: runtime} + + initialImages, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + if err != nil { + t.Fatal(err) + } + + containerID, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + err = srv.ContainerTag(imageID, "test", "0.1", false) + if err != nil { + t.Fatal(err) + } + + containerID, err = srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + _, err = srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + images, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 2 { + t.Fatalf("Expected 2 new images, found %d.", len(images)-len(initialImages)) + } + + _, err = srv.ImageDelete(imageID, true) + if err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 1 { + t.Fatalf("Expected 1 new image, found %d.", len(images)-len(initialImages)) + } + + for _, image := range images { + if strings.Contains(unitTestImageID, image.ID) { + continue + } + if image.Repository == "" { + t.Fatalf("Expected tagged image, got untagged one.") + } + } +} From a97d858b2a0d6bc4d044a241036b97fd78f93022 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 26 Jul 2013 10:21:17 -0700 Subject: [PATCH 16/54] Clean up 'manifesto' in docs --- docs/sources/concepts/manifesto.rst | 61 ----------------------------- 1 file changed, 61 deletions(-) diff --git a/docs/sources/concepts/manifesto.rst b/docs/sources/concepts/manifesto.rst index ae09647094..7dd4b4bdda 100644 --- a/docs/sources/concepts/manifesto.rst +++ b/docs/sources/concepts/manifesto.rst @@ -4,10 +4,6 @@ .. _dockermanifesto: -*(This was our original Welcome page, but it is a bit forward-looking -for docs, and maybe not enough vision for a true manifesto. We'll -reveal more vision in the future to make it more Manifesto-y.)* - Docker Manifesto ---------------- @@ -131,60 +127,3 @@ sitting 10 miles away. With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - -Standard Container Specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(TODO) - -Image format -~~~~~~~~~~~~ - -Standard operations -~~~~~~~~~~~~~~~~~~~ - -- Copy -- Run -- Stop -- Wait -- Commit -- Attach standard streams -- List filesystem changes -- ... - -Execution environment -~~~~~~~~~~~~~~~~~~~~~ - -Root filesystem -^^^^^^^^^^^^^^^ - -Environment variables -^^^^^^^^^^^^^^^^^^^^^ - -Process arguments -^^^^^^^^^^^^^^^^^ - -Networking -^^^^^^^^^^ - -Process namespacing -^^^^^^^^^^^^^^^^^^^ - -Resource limits -^^^^^^^^^^^^^^^ - -Process monitoring -^^^^^^^^^^^^^^^^^^ - -Logging -^^^^^^^ - -Signals -^^^^^^^ - -Pseudo-terminal allocation -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Security -^^^^^^^^ - From bdc79ac8b2dfad302f9e144711067a566726cfa2 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:40:55 -0400 Subject: [PATCH 17/54] Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304 --- Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index aadabb8711..7258af5bf7 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -20,6 +20,8 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " + # Listen on all interfaces so that the daemon is accessible from the host + pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From 5eb590e79d081ecfa82ba940ea856c4a2c0411ca Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:44:06 -0400 Subject: [PATCH 18/54] Update AUTHORS --- .mailmap | 2 ++ AUTHORS | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 452ac41d8f..11ff5357d8 100644 --- a/.mailmap +++ b/.mailmap @@ -23,3 +23,5 @@ Thatcher Peskens Walter Stanish Roberto Hashioka +Konstantin Pelykh +David Sissitka diff --git a/AUTHORS b/AUTHORS index 86d03f6e12..c811de6526 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,12 +4,15 @@ # For a list of active project maintainers, see the MAINTAINERS file. # Al Tobey +Alex Gaynor Alexey Shamrin Andrea Luzzardi Andreas Tiefenthaler Andrew Munsell +Andrews Medina Andy Rothfusz Andy Smith +Anthony Bishopric Antony Messerli Barry Allard Brandon Liu @@ -23,17 +26,22 @@ Daniel Gasienica Daniel Mizyrycki Daniel Robinson Daniel Von Fange +Daniel YC Lin +David Sissitka Dominik Honnef Don Spaulding Dr Nic Williams Elias Probst Eric Hanchrow -Evan Wies Eric Myhre +Erno Hopearuoho +Evan Wies ezbercih +Fabrizio Regini Flavio Castelli Francisco Souza Frederick F. Kautz IV +Gabriel Monroy Gareth Rushgrove Guillaume J. Charmes Harley Laue @@ -41,6 +49,7 @@ Hunter Blanks Jeff Lindsay Jeremy Grosser Joffrey F +Johan Euphrosine John Costa Jon Wedaman Jonas Pfenniger @@ -48,28 +57,39 @@ Jonathan Rudenberg Joseph Anthony Pasquale Holsten Julien Barbier Jérôme Petazzoni +Karan Lyons +Keli Hu Ken Cochrane Kevin J. Lynagh kim0 +Kimbro Staken Kiran Gangadharan +Konstantin Pelykh Louis Opter +Marco Hennings Marcus Farkas Mark McGranaghan Maxim Treskin meejah Michael Crosby +Mike Gaffney Mikhail Sobolev +Nan Monnand Deng Nate Jones Nelson Chen Niall O'Higgins +Nick Stenning +Nick Stinemates odk- Paul Bowsher Paul Hammond Phil Spitler Piotr Bogdan Renato Riccieri Santos Zannon +Rhys Hiltner Robert Obryk Roberto Hashioka +Ryan Fowler Sam Alba Sam J Sharpe Shawn Siefkas @@ -83,6 +103,8 @@ Thomas Hansen Tianon Gravi Tim Terhorst Tobias Bieniek +Tobias Schwab +Tom Hulihan unclejack Victor Vieux Vivek Agarwal From b15cfd3530cc228dc065746c9323758c8abb0481 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 26 Jul 2013 14:57:16 -0700 Subject: [PATCH 19/54] - Builder: Create directories with 755 instead of 700 within ADD instruction --- buildfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildfile.go b/buildfile.go index 75ebdd7a7c..a25b86d794 100644 --- a/buildfile.go +++ b/buildfile.go @@ -242,7 +242,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } else if err := UntarPath(origPath, destPath); err != nil { utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file - if err := os.MkdirAll(path.Dir(destPath), 0700); err != nil { + if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := CopyWithTar(origPath, destPath); err != nil { From 2d85a20c71c9418301c3161c6853a07fca612676 Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 18:29:27 -0700 Subject: [PATCH 20/54] Add required go version for compilation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96da13feaf..5e03b2993c 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ Installing from source ---------------------- 1. Make sure you have a [Go language](http://golang.org/doc/install) -compiler and [git](http://git-scm.com) installed. +compiler >= 1.1 and [git](http://git-scm.com) installed. 2. Checkout the source code ```bash From d4f70397930b15c8860a71df1abb9bc5ac9ab4ff Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:00:36 -0700 Subject: [PATCH 21/54] Do not show empty parenthesis if the default configuration is missing. --- commands.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 0fabaa385f..a7df369e5f 100644 --- a/commands.go +++ b/commands.go @@ -314,13 +314,21 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) + var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + if configDefault == "" { + fmt.Fprintf(cli.out, "%s: ", prompt) + } else { + fmt.Fprintf(cli.out, "%s (%s): ", prompt, configDefault) + } + } + authconfig, ok := cli.configFile.Configs[auth.IndexServerAddress()] if !ok { authconfig = auth.AuthConfig{} } if *flUsername == "" { - fmt.Fprintf(cli.out, "Username (%s): ", authconfig.Username) + promptDefault(cli.out, "Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -340,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - fmt.Fprintf(cli.out, "Email (%s): ", authconfig.Email) + promptDefault(cli.out, "Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 88b6ea993d76fb8891ee7a7fa8828b5c5753c7f5 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:17:57 -0700 Subject: [PATCH 22/54] Remove unused argument. --- commands.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index a7df369e5f..7288b58229 100644 --- a/commands.go +++ b/commands.go @@ -314,7 +314,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) - var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + var promptDefault = func(prompt string, configDefault string) { if configDefault == "" { fmt.Fprintf(cli.out, "%s: ", prompt) } else { @@ -328,7 +328,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flUsername == "" { - promptDefault(cli.out, "Username", authconfig.Username) + promptDefault("Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -348,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - promptDefault(cli.out, "Email", authconfig.Email) + promptDefault("Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 97a2dc96f23b25bc7980d2a4514b7065ff4edb9e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 28 Jul 2013 12:57:09 -0700 Subject: [PATCH 23/54] Remove deprecated copy from README --- README.md | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/README.md b/README.md index 5e03b2993c..5eee1a3db6 100644 --- a/README.md +++ b/README.md @@ -444,51 +444,6 @@ With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - - -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - ### Legal Transfers of Docker shall be in accordance with applicable export From 63876e7dbdd372745855345632af62b8cd976dfa Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:15:27 +0000 Subject: [PATCH 24/54] use ParseRepositoryTag instead on split on : in imagedelete --- server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server.go b/server.go index ce1fc8eaf8..f416d8f06d 100644 --- a/server.go +++ b/server.go @@ -1037,13 +1037,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - var tag string - if strings.Contains(name, ":") { - nameParts := strings.Split(name, ":") - name = nameParts[0] - tag = nameParts[1] - } - + name, tag := utils.ParseRepositoryTag(name) return srv.deleteImage(img, name, tag) } From 3852d0599097581d5dc0bcfcb7aa010d564beb9a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:01 +0000 Subject: [PATCH 25/54] add ParseRepositoryTag tests --- utils/utils.go | 7 +++---- utils/utils_test.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool diff --git a/utils/utils_test.go b/utils/utils_test.go index 5caa809f67..5c480b9438 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -282,3 +282,24 @@ func TestParseHost(t *testing.T) { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } } + +func TestParseRepositoryTag(t *testing.T) { + if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) + } +} From bb241c10e2f124406f00c8e40e00fca67934638a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:14 +0000 Subject: [PATCH 26/54] add regression tests --- server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server_test.go b/server_test.go index 8612b3fcea..0caf8a5f24 100644 --- a/server_test.go +++ b/server_test.go @@ -20,7 +20,7 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestContainerTagImageDelete(t *testing.T) { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { t.Fatal(err) } From b2aa877bf0a3f140516d1c17117369b2194f2b91 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 16:40:35 +0000 Subject: [PATCH 27/54] fix #1314 discard error when loading old container format --- container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index d0b6ca4ce2..cbcd3f17fa 100644 --- a/container.go +++ b/container.go @@ -266,7 +266,8 @@ func (container *Container) FromDisk() error { return err } // Load container settings - if err := json.Unmarshal(data, container); err != nil { + // udp broke compat of docker.PortMapping, but it's not used when loading a container, we can skip it + if err := json.Unmarshal(data, container); err != nil && !strings.Contains(err.Error(), "docker.PortMapping") { return err } return nil From 17ffb0ac84874656a61a477fa29048cb82b28748 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 29 Jul 2013 09:45:19 -0700 Subject: [PATCH 28/54] testing, issue #1331: Add registry functional test to docker-ci --- testing/README.rst | 4 ++++ testing/Vagrantfile | 4 +++- testing/buildbot/credentials.cfg | 5 +++++ testing/buildbot/master.cfg | 20 ++++++++++++++++++-- testing/buildbot/requirements.txt | 1 + testing/buildbot/setup_credentials.sh | 17 +++++++++++++++++ testing/functionaltests/test_registry.sh | 11 +++++++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 testing/buildbot/credentials.cfg create mode 100755 testing/buildbot/setup_credentials.sh create mode 100755 testing/functionaltests/test_registry.sh diff --git a/testing/README.rst b/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/testing/README.rst +++ b/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/testing/Vagrantfile b/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/testing/buildbot/credentials.cfg b/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/testing/buildbot/master.cfg b/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/testing/buildbot/master.cfg +++ b/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/testing/buildbot/requirements.txt b/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/testing/buildbot/requirements.txt +++ b/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/testing/buildbot/setup_credentials.sh b/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/testing/functionaltests/test_registry.sh b/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow From c8ec36d1b9bfbe1e22acd0124409ecb5a109d406 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 10:28:41 -0700 Subject: [PATCH 29/54] Remove unnecessary signal conditional. --- commands.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index 0fabaa385f..b3e7976995 100644 --- a/commands.go +++ b/commands.go @@ -1658,14 +1658,11 @@ func (cli *DockerCli) monitorTtySize(id string) error { } cli.resizeTty(id) - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGWINCH) + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGWINCH) go func() { - for sig := range c { - if sig == syscall.SIGWINCH { - cli.resizeTty(id) - } - } + <-sigchan + cli.resizeTty(id) }() return nil } From f7542664e3efb35b6b153dc2b2cc9cdac181989a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 29 Jul 2013 11:03:09 -0700 Subject: [PATCH 30/54] Make sure ADD will create everything in 0755 --- archive.go | 2 +- docs/sources/use/builder.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/archive.go b/archive.go index 01af86006f..bb79cd34d4 100644 --- a/archive.go +++ b/archive.go @@ -173,7 +173,7 @@ func CopyWithTar(src, dst string) error { } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) - if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index aa2fd6b92a..9f5ee8b795 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -182,7 +182,7 @@ The copy obeys the following rules: written at ````. * If ```` doesn't exist, it is created along with all missing directories in its path. All new files and directories are created - with mode 0700, uid and gid 0. + with mode 0755, uid and gid 0. 3.8 ENTRYPOINT -------------- From 10e37198aa14cb3192fd0bf29572a5dce58c348f Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 11:13:59 -0700 Subject: [PATCH 31/54] Keep the loop to allow resizing more than once. --- commands.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index b3e7976995..b4853e967d 100644 --- a/commands.go +++ b/commands.go @@ -1661,8 +1661,10 @@ func (cli *DockerCli) monitorTtySize(id string) error { sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGWINCH) go func() { - <-sigchan - cli.resizeTty(id) + for { + <-sigchan + cli.resizeTty(id) + } }() return nil } From 8ca7b0646e6c4346075656f46847f53c2e868a3d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 17 Jul 2013 12:13:22 -0700 Subject: [PATCH 32/54] Refactor checksum --- auth/auth.go | 4 +- graph.go | 61 ++--------------- image.go | 99 --------------------------- registry/registry.go | 58 +++++++++++++--- server.go | 96 +++++---------------------- utils/tarsum.go | 155 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 247 deletions(-) create mode 100644 utils/tarsum.go diff --git a/auth/auth.go b/auth/auth.go index 39de876875..e402031ca2 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" +//const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/" +const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/graph.go b/graph.go index 42d1bdbd4c..eea2cbec8a 100644 --- a/graph.go +++ b/graph.go @@ -1,9 +1,7 @@ package docker import ( - "encoding/json" "fmt" - "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -11,17 +9,13 @@ import ( "path" "path/filepath" "strings" - "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *utils.TruncIndex - checksumLock map[string]*sync.Mutex - lockSumFile *sync.Mutex - lockSumMap *sync.Mutex + Root string + idIndex *utils.TruncIndex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -36,11 +30,8 @@ func NewGraph(root string) (*Graph, error) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: utils.NewTruncIndex(), - checksumLock: make(map[string]*sync.Mutex), - lockSumFile: &sync.Mutex{}, - lockSumMap: &sync.Mutex{}, + Root: abspath, + idIndex: utils.NewTruncIndex(), } if err := graph.restore(); err != nil { return nil, err @@ -99,11 +90,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, err } } - graph.lockSumMap.Lock() - defer graph.lockSumMap.Unlock() - if _, exists := graph.checksumLock[img.ID]; !exists { - graph.checksumLock[img.ID] = &sync.Mutex{} - } return img, nil } @@ -126,7 +112,6 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, layerData != nil, img); err != nil { return nil, err } - go img.Checksum() return img, nil } @@ -154,7 +139,6 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { } img.graph = graph graph.idIndex.Add(img.ID) - graph.checksumLock[img.ID] = &sync.Mutex{} return nil } @@ -311,40 +295,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) getStoredChecksums() (map[string]string, error) { - checksums := make(map[string]string) - // FIXME: Store the checksum in memory - - if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil { - if err := json.Unmarshal(checksumDict, &checksums); err != nil { - return nil, err - } - } - return checksums, nil -} - -func (graph *Graph) storeChecksums(checksums map[string]string) error { - checksumJSON, err := json.Marshal(checksums) - if err != nil { - return err - } - if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJSON, 0600); err != nil { - return err - } - return nil -} - -func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error { - graph.lockSumFile.Lock() - defer graph.lockSumFile.Unlock() - - localChecksums, err := graph.getStoredChecksums() - if err != nil { - return err - } - for id, elem := range newChecksums { - localChecksums[id] = elem.Checksum - } - return graph.storeChecksums(localChecksums) -} diff --git a/image.go b/image.go index 5240ec776f..dd066f88cd 100644 --- a/image.go +++ b/image.go @@ -2,7 +2,6 @@ package docker import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -72,26 +71,6 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { return err } - if store { - layerArchive := layerArchivePath(root) - file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - // FIXME: Retrieve the image layer size from here? - if _, err := io.Copy(file, layerData); err != nil { - return err - } - // FIXME: Don't close/open, read/write instead of Copy - file.Close() - - file, err = os.Open(layerArchive) - if err != nil { - return err - } - defer file.Close() - layerData = file - } // If layerData is not nil, unpack it into the new layer if layerData != nil { start := time.Now() @@ -128,10 +107,6 @@ func layerPath(root string) string { return path.Join(root, "layer") } -func layerArchivePath(root string) string { - return path.Join(root, "layer.tar.xz") -} - func jsonPath(root string) string { return path.Join(root, "json") } @@ -308,80 +283,6 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) Checksum() (string, error) { - img.graph.checksumLock[img.ID].Lock() - defer img.graph.checksumLock[img.ID].Unlock() - - root, err := img.root() - if err != nil { - return "", err - } - - checksums, err := img.graph.getStoredChecksums() - if err != nil { - return "", err - } - if checksum, ok := checksums[img.ID]; ok { - return checksum, nil - } - - layer, err := img.layer() - if err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) - if err != nil { - return "", err - } - - var layerData io.Reader - - if file, err := os.Open(layerArchivePath(root)); err != nil { - if os.IsNotExist(err) { - layerData, err = Tar(layer, Xz) - if err != nil { - return "", err - } - } else { - return "", err - } - } else { - defer file.Close() - layerData = file - } - - h := sha256.New() - if _, err := h.Write(jsonData); err != nil { - return "", err - } - if _, err := h.Write([]byte("\n")); err != nil { - return "", err - } - - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - - // Reload the json file to make sure not to overwrite faster sums - img.graph.lockSumFile.Lock() - defer img.graph.lockSumFile.Unlock() - - checksums, err = img.graph.getStoredChecksums() - if err != nil { - return "", err - } - - checksums[img.ID] = hash - - // Dump the checksums to disc - if err := img.graph.storeChecksums(checksums); err != nil { - return hash, err - } - - return hash, nil -} - func (img *Image) getParentsSize(size int64) int64 { parentImage, err := img.GetParent() if err != nil || parentImage == nil { diff --git a/registry/registry.go b/registry/registry.go index adef1c7baa..cac77ba049 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -330,16 +330,52 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e }, nil } +func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - // FIXME: try json with UTF8 + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) r.setUserAgent(req) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) @@ -364,10 +400,14 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { - return err + return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} @@ -375,18 +415,18 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) + return "", fmt.Errorf("Failed to upload layer: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return nil + return tarsumLayer.Sum(), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { diff --git a/server.go b/server.go index ce1fc8eaf8..6944df315f 100644 --- a/server.go +++ b/server.go @@ -455,12 +455,6 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName return err } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err - } - utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { @@ -598,41 +592,6 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return nil } -// Retrieve the checksum of an image -// Priority: -// - Check on the stored checksums -// - Check if the archive exists, if it does not, ask the registry -// - If the archive does exists, process the checksum from it -// - If the archive does not exists and not found on registry, process checksum from layer -func (srv *Server) getChecksum(imageID string) (string, error) { - // FIXME: Use in-memory map instead of reading the file each time - if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { - return "", err - } else if checksum, exists := sums[imageID]; exists { - return checksum, nil - } - - img, err := srv.runtime.graph.Get(imageID) - if err != nil { - return "", err - } - - if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil { - if os.IsNotExist(err) { - // TODO: Ask the registry for the checksum - // As the archive is not there, it is supposed to come from a pull. - } else { - return "", err - } - } - - checksum, err := img.Checksum() - if err != nil { - return "", err - } - return checksum, nil -} - // Retrieve the all the images to be uploaded in the correct order // Note: we can't use a map as it is not ordered func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) { @@ -649,14 +608,10 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return nil } imageSet[img.ID] = struct{}{} - checksum, err := srv.getChecksum(img.ID) - if err != nil { - return err - } + imgList = append([]*registry.ImgData{{ - ID: img.ID, - Checksum: checksum, - Tag: tag, + ID: img.ID, + Tag: tag, }}, imgList...) return nil }) @@ -666,7 +621,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - out.Write(sf.FormatStatus("Processing checksums")) + imgList, err := srv.getImageList(localRepo) if err != nil { return err @@ -716,14 +671,8 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } out.Write(sf.FormatStatus("Pushing %s", imgID)) - // Make sure we have the image's checksum - checksum, err := srv.getChecksum(imgID) - if err != nil { - return err - } imgData := ®istry.ImgData{ - ID: imgID, - Checksum: checksum, + ID: imgID, } // Send the json @@ -735,36 +684,23 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, return err } - // Retrieve the tarball to be sent - var layerData *TempArchive - // If the archive exists, use it - file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID))) + layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - if os.IsNotExist(err) { - // If the archive does not exist, create one from the layer - layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - } else { - return err - } - } else { - defer file.Close() - st, err := file.Stat() - if err != nil { - return err - } - layerData = &TempArchive{ - File: file, - Size: st.Size(), - } + return fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + return err + } else { + imgData.Checksum = checksum + } + + // Send the checksum + if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { return err } + return nil } diff --git a/utils/tarsum.go b/utils/tarsum.go new file mode 100644 index 0000000000..0fd5ac106b --- /dev/null +++ b/utils/tarsum.go @@ -0,0 +1,155 @@ +package utils + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "hash" + "io" + "sort" + "strconv" +) + +type verboseHash struct { + hash.Hash +} + +func (h verboseHash) Write(buf []byte) (int, error) { + Debugf("--->%s<---", buf) + return h.Hash.Write(buf) +} + +type TarSum struct { + io.Reader + tarR *tar.Reader + tarW *tar.Writer + gz *gzip.Writer + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + h2 verboseHash + sums []string + finished bool + first bool +} + +func (ts *TarSum) encodeHeader(h *tar.Header) error { + for _, elem := range [][2]string{ + {"name", h.Name}, + {"mode", strconv.Itoa(int(h.Mode))}, + {"uid", strconv.Itoa(h.Uid)}, + {"gid", strconv.Itoa(h.Gid)}, + {"size", strconv.Itoa(int(h.Size))}, + {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"typeflag", string([]byte{h.Typeflag})}, + {"linkname", h.Linkname}, + {"uname", h.Uname}, + {"gname", h.Gname}, + {"devmajor", strconv.Itoa(int(h.Devmajor))}, + {"devminor", strconv.Itoa(int(h.Devminor))}, + // {"atime", strconv.Itoa(int(h.AccessTime.UTC().Unix()))}, + // {"ctime", strconv.Itoa(int(h.ChangeTime.UTC().Unix()))}, + } { + // Debugf("-->%s<-- -->%s<--", elem[0], elem[1]) + if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { + return err + } + } + return nil +} + +func (ts *TarSum) Read(buf []byte) (int, error) { + if ts.gz == nil { + ts.bufTar = bytes.NewBuffer([]byte{}) + ts.bufGz = bytes.NewBuffer([]byte{}) + ts.tarR = tar.NewReader(ts.Reader) + ts.tarW = tar.NewWriter(ts.bufTar) + ts.gz = gzip.NewWriter(ts.bufGz) + ts.h = sha256.New() + // ts.h = verboseHash{sha256.New()} + ts.h.Reset() + ts.first = true + } + + if ts.finished { + return ts.bufGz.Read(buf) + } + buf2 := make([]byte, len(buf), cap(buf)) + + n, err := ts.tarR.Read(buf2) + if err != nil { + if err == io.EOF { + if _, err := ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + if !ts.first { + ts.sums = append(ts.sums, hex.EncodeToString(ts.h.Sum(nil))) + ts.h.Reset() + } else { + ts.first = false + } + + currentHeader, err := ts.tarR.Next() + if err != nil { + if err == io.EOF { + if err := ts.gz.Close(); err != nil { + return 0, err + } + ts.finished = true + return n, nil + } + return n, err + } + if err := ts.encodeHeader(currentHeader); err != nil { + return 0, err + } + if err := ts.tarW.WriteHeader(currentHeader); err != nil { + return 0, err + } + if _, err := ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + if _, err := io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) + } + return n, err + } + + // Filling the hash buffer + if _, err = ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + + // Filling the tar writter + if _, err = ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + + // Filling the gz writter + if _, err = io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) +} + +func (ts *TarSum) Sum() string { + sort.Strings(ts.sums) + h := sha256.New() + for _, sum := range ts.sums { + Debugf("-->%s<--", sum) + h.Write([]byte(sum)) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + Debugf("checksum processed: %s", checksum) + return checksum +} From e3f68b22d8f0635a8c08ab56721e56dbe570a49a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 14:50:32 -0700 Subject: [PATCH 33/54] Handle extra-paremeter within checksum calculations --- registry/registry.go | 17 +++++++++++------ server.go | 2 +- utils/tarsum.go | 7 +++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index cac77ba049..40b9872a4a 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -17,8 +17,10 @@ import ( "strings" ) -var ErrAlreadyExists = errors.New("Image already exists") -var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +var ( + ErrAlreadyExists = errors.New("Image already exists") + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +) func pingRegistryEndpoint(endpoint string) error { if endpoint == auth.IndexServerAddress() { @@ -266,8 +268,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err @@ -378,7 +383,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) r.setUserAgent(req) - utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -400,11 +404,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err @@ -426,7 +431,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return tarsumLayer.Sum(), nil + return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { @@ -474,7 +479,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("PUT %s", u) + utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { diff --git a/server.go b/server.go index 6944df315f..7309279805 100644 --- a/server.go +++ b/server.go @@ -690,7 +690,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { return err } else { imgData.Checksum = checksum diff --git a/utils/tarsum.go b/utils/tarsum.go index 0fd5ac106b..015b9b3076 100644 --- a/utils/tarsum.go +++ b/utils/tarsum.go @@ -142,14 +142,17 @@ func (ts *TarSum) Read(buf []byte) (int, error) { return ts.bufGz.Read(buf) } -func (ts *TarSum) Sum() string { +func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + if extra != nil { + h.Write(extra) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum } From 0badda9f1587c11a13dca17c68b30addd757237c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:40:33 -0700 Subject: [PATCH 34/54] Refactor the image size storage --- graph.go | 6 +++--- image.go | 43 ++++++++++++++++++++++++++++++++++--------- server.go | 2 +- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/graph.go b/graph.go index eea2cbec8a..1fe14f6458 100644 --- a/graph.go +++ b/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { return err } // Commit diff --git a/image.go b/image.go index dd066f88cd..7f03cc4bf8 100644 --- a/image.go +++ b/image.go @@ -13,6 +13,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "time" ) @@ -46,6 +47,19 @@ func LoadImage(root string) (*Image, error) { if err := ValidateID(img.ID); err != nil { return nil, err } + + if buf, err := ioutil.ReadFile(path.Join(root, "layersize")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + if size, err := strconv.Atoi(string(buf)); err != nil { + return nil, err + } else { + img.Size = int64(size) + } + } + // Check that the filesystem layer exists if stat, err := os.Stat(layerPath(root)); err != nil { if os.IsNotExist(err) { @@ -58,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) @@ -81,25 +95,36 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds()) } + // If raw json is provided, then use it + if jsonData != nil { + return ioutil.WriteFile(jsonPath(root), jsonData, 0600) + } else { // Otherwise, unmarshal the image + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } + return StoreSize(img, root) } func StoreSize(img *Image, root string) error { layer := layerPath(root) + var totalSize int64 = 0 filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error { - img.Size += fileInfo.Size() + totalSize += fileInfo.Size() return nil }) + img.Size = totalSize - // Store the json ball - jsonData, err := json.Marshal(img) - if err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err + if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil { + return nil } + return nil } diff --git a/server.go b/server.go index 7309279805..de73d6c815 100644 --- a/server.go +++ b/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { return err } } From 0f134b4bf81a4d0160932852854b190b7ee7e3b9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:44:55 -0700 Subject: [PATCH 35/54] Remove unused parameter --- graph.go | 6 +++--- image.go | 2 +- server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/graph.go b/graph.go index 1fe14f6458..bf27f93ed4 100644 --- a/graph.go +++ b/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp); err != nil { return err } // Commit diff --git a/image.go b/image.go index 7f03cc4bf8..220f1c70a5 100644 --- a/image.go +++ b/image.go @@ -72,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) diff --git a/server.go b/server.go index de73d6c815..aecc730794 100644 --- a/server.go +++ b/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), img); err != nil { return err } } From 394941b6b0a30fecf8ae7b6de5880fa553141f93 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:16:31 -0700 Subject: [PATCH 36/54] Switch json/payload order --- auth/auth.go | 4 ++-- utils/tarsum.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index e402031ca2..6dd6ceb620 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -//const INDEXSERVER = "https://index.docker.io/v1/" +const INDEXSERVER = "https://index.docker.io/v1/" -const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" +//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/utils/tarsum.go b/utils/tarsum.go index 015b9b3076..d3e1db61f1 100644 --- a/utils/tarsum.go +++ b/utils/tarsum.go @@ -145,13 +145,13 @@ func (ts *TarSum) Read(buf []byte) (int, error) { func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() + if extra != nil { + h.Write(extra) + } for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - if extra != nil { - h.Write(extra) - } checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum From 5b27652ac6eaf1bc4c2a16e51919ec4272a58fd6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:44:34 -0700 Subject: [PATCH 37/54] Make sure the index also receives the checksums --- registry/registry.go | 14 +++++++++++++- server.go | 22 ++++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 40b9872a4a..4e9dd8895f 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -469,7 +469,19 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - imgListJSON, err := json.Marshal(imgList) + cleanImgList := []*ImgData{} + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) if err != nil { return nil, err } diff --git a/server.go b/server.go index aecc730794..88ff2ac3a6 100644 --- a/server.go +++ b/server.go @@ -645,9 +645,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { + if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err + } else { + elem.Checksum = checksum } out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { @@ -663,11 +665,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName return nil } -func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) + return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } out.Write(sf.FormatStatus("Pushing %s", imgID)) @@ -679,29 +681,29 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) - return nil + return "", nil } - return err + return "", err } layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) + return "", fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { - return err + return "", err } else { imgData.Checksum = checksum } // Send the checksum if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { - return err + return "", err } - return nil + return imgData.Checksum, nil } // FIXME: Allow to interupt current push when new push of same image is done. @@ -739,7 +741,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo var token []string out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) - if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { + if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil From 5dc86d7bca17c2996264a18cc26f06d30e532588 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Mon, 29 Jul 2013 14:17:15 -0700 Subject: [PATCH 38/54] Updated the description of run -d The goal is to make it more clear this will give you the container id after run completes. Since stdout is now standard on run, "docker run -d" is the best (or only) way to get the container ID returned from docker after a plain run, but the description (help) does not hint any such thing. --- container.go | 2 +- docs/sources/commandline/command/run.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container.go b/container.go index d0b6ca4ce2..d610c3c7d4 100644 --- a/container.go +++ b/container.go @@ -100,7 +100,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, flHostname := cmd.String("h", "", "Container host name") flUser := cmd.String("u", "", "Username or UID") - flDetach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id") flAttach := NewAttachOpts() cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached") diff --git a/docs/sources/commandline/command/run.rst b/docs/sources/commandline/command/run.rst index db67ef0705..db043c3b3f 100644 --- a/docs/sources/commandline/command/run.rst +++ b/docs/sources/commandline/command/run.rst @@ -15,7 +15,7 @@ -a=map[]: Attach to stdin, stdout or stderr. -c=0: CPU shares (relative weight) -cidfile="": Write the container ID to the file - -d=false: Detached mode: leave the container running in the background + -d=false: Detached mode: Run container in the background, print new container id -e=[]: Set environment variables -h="": Container host name -i=false: Keep stdin open even if not attached From 9ba998312de5826f24f693c8518ecda700135b4b Mon Sep 17 00:00:00 2001 From: dsissitka Date: Tue, 30 Jul 2013 01:39:29 -0400 Subject: [PATCH 39/54] Fixed a couple of minor syntax errors. --- docs/sources/examples/couchdb_data_volumes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/examples/couchdb_data_volumes.rst b/docs/sources/examples/couchdb_data_volumes.rst index d6babe557f..97af733a82 100644 --- a/docs/sources/examples/couchdb_data_volumes.rst +++ b/docs/sources/examples/couchdb_data_volumes.rst @@ -39,7 +39,7 @@ This time, we're requesting shared access to $COUCH1's volumes. .. code-block:: bash - COUCH2=$(docker run -d -volumes-from $COUCH1) shykes/couchdb:2013-05-03) + COUCH2=$(docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03) Browse data on the second database ---------------------------------- @@ -48,6 +48,6 @@ Browse data on the second database HOST=localhost URL="http://$HOST:$(docker port $COUCH2 5984)/_utils/" - echo "Navigate to $URL in your browser. You should see the same data as in the first database!" + echo "Navigate to $URL in your browser. You should see the same data as in the first database"'!' Congratulations, you are running 2 Couchdb containers, completely isolated from each other *except* for their data. From 7d0b8c726c0d178291063751ac735d6caebfb45a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:47:29 +0200 Subject: [PATCH 40/54] add ufw doc --- docs/sources/installation/ubuntulinux.rst | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index ed592d3a9d..299b7a29e9 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -19,6 +19,8 @@ Docker has the following dependencies * Linux kernel 3.8 (read more about :ref:`kernel`) * AUFS file system support (we are working on BTRFS support as an alternative) +Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ + .. _ubuntu_precise: Ubuntu Precise 12.04 (LTS) (64-bit) @@ -135,3 +137,35 @@ Verify it worked **Done!**, now continue with the :ref:`hello_world` example. + + +.. _ufw: + +Docker and UFW +^^^^^^^^^^^^^^ + +Docker uses a bridge to manage containers networking, by default UFW drop all `forwarding`, a first step is to enable forwarding: + +.. code-block:: bash + + sudo nano /etc/default/ufw + ---- + # Change: + # DEFAULT_FORWARD_POLICY="DROP" + # to + DEFAULT_FORWARD_POLICY="ACCEPT" + +Then reload UFW: + +.. code-block:: bash + + sudo ufw reload + + +UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, +you should allow incoming connexions on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + From b14c251862021b2bc82aa8e0146e5e5e80f1c713 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:13:18 +0000 Subject: [PATCH 41/54] fix tests about refactor checksums --- graph_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graph_test.go b/graph_test.go index 18682338d9..2898fccf99 100644 --- a/graph_test.go +++ b/graph_test.go @@ -38,7 +38,7 @@ func TestInterruptedRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - go graph.Register(badArchive, false, image) + go graph.Register(nil, badArchive, image) time.Sleep(200 * time.Millisecond) w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) if _, err := graph.Get(image.ID); err == nil { @@ -49,7 +49,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(goodArchive, false, image); err != nil { + if err := graph.Register(nil, goodArchive, image); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(archive, false, image) + err = graph.Register(nil, archive, image) if err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(archive, false, img1); err != nil { + if err := graph.Register(nil, archive, img1); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { From e4752c8c1a09fc3cc96dbb9be7183b271db3d6b7 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 16:39:35 +0000 Subject: [PATCH 42/54] Add check that the request is good --- utils/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 190dc175cd..def5ae5a50 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -736,6 +736,9 @@ func GetReleaseVersion() string { return "" } defer resp.Body.Close() + if resp.ContentLength > 24 || resp.StatusCode != 200 { + return "" + } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "" From e66e0289abc213164dca1e1eadfb0380b6e81904 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 17:18:19 +0000 Subject: [PATCH 43/54] update http://get.docker.io/latest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6050000582..dd365dc30e 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ release: $(BINRELEASE) s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz s3cmd -P put docker-latest.tgz s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-latest.tgz s3cmd -P put $(SRCRELEASE)/bin/docker s3://get.docker.io/builds/`uname -s`/`uname -m`/docker + echo $(RELEASE_VERSION) > latest ; s3cmd -P put latest s3://get.docker.io/latest ; rm latest srcrelease: $(SRCRELEASE) deps: $(DOCKER_DIR) @@ -65,7 +66,6 @@ $(BINRELEASE): $(SRCRELEASE) rm -f $(BINRELEASE) cd $(SRCRELEASE); make; cp -R bin docker-$(RELEASE_VERSION); tar -f ../$(BINRELEASE) -zv -c docker-$(RELEASE_VERSION) cd $(SRCRELEASE); cp -R bin docker-latest; tar -f ../docker-latest.tgz -zv -c docker-latest - clean: @rm -rf $(dir $(DOCKER_BIN)) ifeq ($(GOPATH), $(BUILD_DIR)) From 3043c2641990d94298c6377b7ef14709263a4709 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 24 Jul 2013 03:01:24 +0000 Subject: [PATCH 44/54] Return registy status code in error Added Details map to the JSONMessage --- api.go | 12 ++++++++---- commands.go | 29 +++++++++++++++++++---------- registry/registry.go | 24 ++++++++++++------------ utils/error.go | 18 ++++++++++++++++++ utils/utils.go | 26 ++++++++++++++++++-------- 5 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 utils/error.go diff --git a/api.go b/api.go index 4ad2ba461a..5869669df0 100644 --- a/api.go +++ b/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } } @@ -472,7 +472,11 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + var code int + if httpErr, ok := err.(*utils.HTTPRequestError); ok { + code = httpErr.StatusCode + } + w.Write(sf.FormatError(err, code)) return nil } return err diff --git a/commands.go b/commands.go index 95ddca1f1d..9ad2c367ae 100644 --- a/commands.go +++ b/commands.go @@ -30,7 +30,8 @@ import ( const VERSION = "0.5.0-dev" var ( - GITCOMMIT string + GITCOMMIT string + AuthRequiredError error = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { @@ -814,10 +815,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged("push"); err != nil { - return err - } - // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -826,13 +823,22 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.configFile.Configs[auth.IndexServerAddress()].Username, name) } - buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) - if err != nil { - return err + v := url.Values{} + push := func() error { + buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) + if err != nil { + return err + } + + return cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out) } - v := url.Values{} - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { + if err := push(); err != nil { + if err == AuthRequiredError { + if err = cli.checkIfLogged("push"); err == nil { + return push() + } + } return err } return nil @@ -1559,6 +1565,9 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } else if err != nil { return err } + if jm.Error != nil && jm.Error.Code == 401 { + return AuthRequiredError + } jm.Display(out) } } else { diff --git a/registry/registry.go b/registry/registry.go index 4e9dd8895f..ed6f4c7df8 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -147,7 +147,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -197,7 +197,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -289,12 +289,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -391,7 +391,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -399,7 +399,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil } @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } @@ -463,7 +463,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -540,7 +540,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -564,7 +564,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -586,7 +586,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { diff --git a/utils/error.go b/utils/error.go new file mode 100644 index 0000000000..7e3c846ebc --- /dev/null +++ b/utils/error.go @@ -0,0 +1,18 @@ +package utils + +import ( + "net/http" +) + +type HTTPRequestError struct { + Message string + StatusCode int +} + +func (e *HTTPRequestError) Error() string { + return e.Message +} + +func NewHTTPRequestError(msg string, resp *http.Response) error { + return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} +} diff --git a/utils/utils.go b/utils/utils.go index c70e80b72e..659d960d0a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -607,12 +607,22 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + Status string `json:"status,omitempty"` + Progress string `json:"progress,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message } func (jm *JSONMessage) Display(out io.Writer) error { @@ -621,8 +631,8 @@ func (jm *JSONMessage) Display(out io.Writer) error { } if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != "" { - return fmt.Errorf(jm.Error) + } else if jm.Error != nil { + return jm.Error } else if jm.ID != "" { fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { @@ -656,7 +666,7 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil { + if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") From 73c6d9f135493220f034d440d26fedc0242e133a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 07:56:53 +0000 Subject: [PATCH 45/54] improve tests --- server_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/server_test.go b/server_test.go index 0caf8a5f24..de95d743ba 100644 --- a/server_test.go +++ b/server_test.go @@ -20,7 +20,11 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { + + if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + t.Fatal(err) + } + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -29,11 +33,24 @@ func TestContainerTagImageDelete(t *testing.T) { t.Fatal(err) } + if len(images) != len(initialImages)+3 { + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + } + + if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + if len(images) != len(initialImages)+2 { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { t.Fatal(err) } From a7068510a5ee4af6776221ba00bc332266f97088 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 08:01:20 +0000 Subject: [PATCH 46/54] fix same issue in api.go --- api.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api.go b/api.go index 834c41a68c..d0a8ada249 100644 --- a/api.go +++ b/api.go @@ -786,12 +786,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") - tag := "" - if strings.Contains(repoName, ":") { - remoteParts := strings.Split(repoName, ":") - tag = remoteParts[1] - repoName = remoteParts[0] - } + repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader From 9a604acc23d30d00ae907acdf756cbcdf0e4cf83 Mon Sep 17 00:00:00 2001 From: Nolan Date: Tue, 30 Jul 2013 13:23:34 -0500 Subject: [PATCH 47/54] Add hostname to the container environment. --- container.go | 1 + 1 file changed, 1 insertion(+) diff --git a/container.go b/container.go index d610c3c7d4..ccc7ab3e9f 100644 --- a/container.go +++ b/container.go @@ -652,6 +652,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-e", "HOME=/", "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "-e", "container=lxc", + "-e", "HOSTNAME="+container.Config.Hostname, ) for _, elem := range container.Config.Env { From e0c24ccfc37b64eb919c5675d8a8f383201fa7cb Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Wed, 31 Jul 2013 12:17:42 -0700 Subject: [PATCH 48/54] Solved the logo being squished in Safari --- docs/theme/docker/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 0b22f22fab..ca26f44dc0 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -79,7 +79,7 @@
- +
From 2424480e2ce14d848c041c53cc041aab91308081 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Fri, 2 Aug 2013 12:24:38 +0700 Subject: [PATCH 49/54] Move note about officially supported kernel It seems this a general note about kernel issues and not specific to Cgroups or namespaces. --- docs/sources/installation/kernel.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/sources/installation/kernel.rst b/docs/sources/installation/kernel.rst index 58730f8191..7c5715a62d 100644 --- a/docs/sources/installation/kernel.rst +++ b/docs/sources/installation/kernel.rst @@ -15,12 +15,11 @@ In short, Docker has the following kernel requirements: - Cgroups and namespaces must be enabled. - - The officially supported kernel is the one recommended by the - :ref:`ubuntu_linux` installation path. It is the one that most developers - will use, and the one that receives the most attention from the core - contributors. If you decide to go with a different kernel and hit a bug, - please try to reproduce it with the official kernels first. +The officially supported kernel is the one recommended by the +:ref:`ubuntu_linux` installation path. It is the one that most developers +will use, and the one that receives the most attention from the core +contributors. If you decide to go with a different kernel and hit a bug, +please try to reproduce it with the official kernels first. If you cannot or do not want to use the "official" kernels, here is some technical background about the features (both optional and From 3a123bc479457c4dfa14e39b7c42d9a9dccf8c32 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 16:18:54 +0000 Subject: [PATCH 50/54] Add no cache for docker build Add a new flag to disable the image cache when building images. --- api.go | 7 ++- buildfile.go | 55 ++++++++++++--------- buildfile_test.go | 2 +- commands.go | 4 ++ docs/sources/api/docker_remote_api_v1.4.rst | 1 + docs/sources/commandline/command/build.rst | 1 + 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/api.go b/api.go index b8b7897c32..95b9c98de8 100644 --- a/api.go +++ b/api.go @@ -793,6 +793,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") + rawNoCache := r.FormValue("nocache") repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader @@ -839,8 +840,12 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if err != nil { return err } + noCache, err := getBoolParam(rawNoCache) + if err != nil { + return err + } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache) id, err := b.Build(context) if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) diff --git a/buildfile.go b/buildfile.go index 736725e915..159d7ba704 100644 --- a/buildfile.go +++ b/buildfile.go @@ -26,11 +26,12 @@ type buildFile struct { builder *Builder srv *Server - image string - maintainer string - config *Config - context string - verbose bool + image string + maintainer string + config *Config + context string + verbose bool + utilizeCache bool tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -94,15 +95,17 @@ func (b *buildFile) CmdRun(args string) error { utils.Debugf("Command to be executed: %v", b.config.Cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } cid, err := b.run() @@ -397,16 +400,19 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } + container, err := b.builder.Create(b.config) if err != nil { return err @@ -500,7 +506,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache bool) BuildFile { return &buildFile{ builder: NewBuilder(srv.runtime), runtime: srv.runtime, @@ -510,5 +516,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, + utilizeCache: utilizeCache, } } diff --git a/buildfile_test.go b/buildfile_test.go index 78e53b8419..eda047d8ea 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -227,7 +227,7 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { ip := runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false) + buildfile := NewBuildFile(srv, ioutil.Discard, false, true) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) diff --git a/commands.go b/commands.go index 95ddca1f1d..7a1935ee61 100644 --- a/commands.go +++ b/commands.go @@ -160,6 +160,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") suppressOutput := cmd.Bool("q", false, "Suppress verbose build output") + noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image") if err := cmd.Parse(args); err != nil { return nil @@ -208,6 +209,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if isRemote { v.Set("remote", cmd.Arg(0)) } + if *noCache { + v.Set("nocache", "1") + } req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err diff --git a/docs/sources/api/docker_remote_api_v1.4.rst b/docs/sources/api/docker_remote_api_v1.4.rst index 6ee0b35fa2..6830bacde0 100644 --- a/docs/sources/api/docker_remote_api_v1.4.rst +++ b/docs/sources/api/docker_remote_api_v1.4.rst @@ -928,6 +928,7 @@ Build an image from Dockerfile via stdin :query t: tag to be applied to the resulting image in case of success :query q: suppress verbose build output + :query nocache: do not use the cache when building the image :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst index 45b6d2ec8e..fcd78dd1ab 100644 --- a/docs/sources/commandline/command/build.rst +++ b/docs/sources/commandline/command/build.rst @@ -12,6 +12,7 @@ Build a new container image from the source code at PATH -t="": Tag to be applied to the resulting image in case of success. -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context From b9f06959244e3f77eb212c7c234b06eb7b750999 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 19:12:38 +0000 Subject: [PATCH 51/54] Add unit tests for build no cache --- buildfile_test.go | 102 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/buildfile_test.go b/buildfile_test.go index eda047d8ea..0e2d9ecefc 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -195,21 +195,23 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t) + buildImage(ctx, t, nil, true) } } -func buildImage(context testContextTemplate, t *testing.T) *Image { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image { + if srv == nil { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), + srv = &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } } httpServer, err := mkTestingFileServer(context.remoteFiles) @@ -224,10 +226,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { } port := httpServer.URL[idx+1:] - ip := runtime.networkManager.bridgeNetwork.IP + ip := srv.runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false, true) + buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) @@ -245,7 +247,7 @@ func TestVolume(t *testing.T) { from {IMAGE} volume /test cmd Hello world - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if len(img.Config.Volumes) == 0 { t.Fail() @@ -261,7 +263,7 @@ func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if img.Author != "dockerio" { t.Fail() @@ -273,7 +275,7 @@ func TestBuildEnv(t *testing.T) { from {IMAGE} env port 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) hasEnv := false for _, envVar := range img.Config.Env { if envVar == "port=4243" { @@ -291,7 +293,7 @@ func TestBuildCmd(t *testing.T) { from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -308,7 +310,7 @@ func TestBuildExpose(t *testing.T) { from {IMAGE} expose 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -320,8 +322,70 @@ func TestBuildEntrypoint(t *testing.T) { from {IMAGE} entrypoint ["/bin/echo"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Entrypoint[0] != "/bin/echo" { } } + +func TestBuildImageWithCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, true) + + if imageId != img.ID { + t.Logf("Image ids should match: %s != %s", imageId, img.ID) + t.Fail() + } +} + +func TestBuildImageWithoutCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, false) + + if imageId == img.ID { + t.Logf("Image ids should not match: %s == %s", imageId, img.ID) + t.Fail() + } +} From 07fee445593982ef9063a184c05ba93ecacd2e65 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 2 Aug 2013 19:18:02 -0300 Subject: [PATCH 52/54] Revert "Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304" This reverts commit bdc79ac8b2dfad302f9e144711067a566726cfa2. --- Vagrantfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7258af5bf7..aadabb8711 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -20,8 +20,6 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " - # Listen on all interfaces so that the daemon is accessible from the host - pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From dde8f74ceae83f26386ec29e42f615fdc7945e80 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 2 Aug 2013 15:58:10 -0700 Subject: [PATCH 53/54] Fix TestEnv --- container_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/container_test.go b/container_test.go index a1ac0bd33a..f29ae9e4ea 100644 --- a/container_test.go +++ b/container_test.go @@ -960,6 +960,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", + "HOSTNAME=" + container.ShortID(), } sort.Strings(goodEnv) if len(goodEnv) != len(actualEnv) { From dae585c6e4c19817b2dbd106171728a0bb564ccc Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 30 Jul 2013 22:48:20 +0000 Subject: [PATCH 54/54] Return JSONError for HTTPResponse error --- api.go | 12 ++++-------- commands.go | 2 +- registry/registry.go | 4 ++-- utils/error.go | 18 ------------------ utils/utils.go | 13 ++++++++++++- utils_test.go | 2 +- 6 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 utils/error.go diff --git a/api.go b/api.go index 5869669df0..4ad2ba461a 100644 --- a/api.go +++ b/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } } @@ -472,11 +472,7 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - var code int - if httpErr, ok := err.(*utils.HTTPRequestError); ok { - code = httpErr.StatusCode - } - w.Write(sf.FormatError(err, code)) + w.Write(sf.FormatError(err)) return nil } return err diff --git a/commands.go b/commands.go index 9ad2c367ae..355e91f4bf 100644 --- a/commands.go +++ b/commands.go @@ -31,7 +31,7 @@ const VERSION = "0.5.0-dev" var ( GITCOMMIT string - AuthRequiredError error = fmt.Errorf("Authentication is required.") + AuthRequiredError = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { diff --git a/registry/registry.go b/registry/registry.go index ed6f4c7df8..5b8480d183 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } diff --git a/utils/error.go b/utils/error.go deleted file mode 100644 index 7e3c846ebc..0000000000 --- a/utils/error.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "net/http" -) - -type HTTPRequestError struct { - Message string - StatusCode int -} - -func (e *HTTPRequestError) Error() string { - return e.Message -} - -func NewHTTPRequestError(msg string, resp *http.Response) error { - return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} -} diff --git a/utils/utils.go b/utils/utils.go index 659d960d0a..2323829f65 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -625,6 +625,13 @@ func (e *JSONError) Error() string { return e.Message } +func NewHTTPRequestError(msg string, res *http.Response) error { + return &JSONError{ + Message: msg, + Code: res.StatusCode, + } +} + func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) @@ -666,7 +673,11 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") diff --git a/utils_test.go b/utils_test.go index c4adeb4a74..91df6183fc 100644 --- a/utils_test.go +++ b/utils_test.go @@ -191,7 +191,7 @@ func TestMergeConfig(t *testing.T) { if len(configUser.Volumes) != 3 { t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes)) } - for v, _ := range configUser.Volumes { + for v := range configUser.Volumes { if v != "/test1" && v != "/test2" && v != "/test3" { t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v) }