From 0f312113d3ce37d57fb28eb98c8abcdcbfcd39a3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sun, 19 May 2013 10:46:24 -0700 Subject: [PATCH 01/12] Move docker build to client --- api.go | 34 +++-- api_params.go | 5 + builder.go | 358 +--------------------------------------------- builder_client.go | 275 +++++++++++++++++++++++++++++++++++ commands.go | 82 ++++++----- server.go | 47 ++++-- utils.go | 39 +++++ 7 files changed, 426 insertions(+), 414 deletions(-) create mode 100644 builder_client.go diff --git a/api.go b/api.go index 8984d00cd9..5cb3da85c2 100644 --- a/api.go +++ b/api.go @@ -370,19 +370,6 @@ func postImagesPush(srv *Server, w http.ResponseWriter, r *http.Request, vars ma return nil } -func postBuild(srv *Server, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - in, out, err := hijackServer(w) - if err != nil { - return err - } - defer in.Close() - fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") - if err := srv.ImageCreateFromFile(in, out); err != nil { - fmt.Fprintf(out, "Error: %s\n", err) - } - return nil -} - func postContainersCreate(srv *Server, w http.ResponseWriter, r *http.Request, vars map[string]string) error { config := &Config{} if err := json.NewDecoder(r.Body).Decode(config); err != nil { @@ -593,6 +580,25 @@ func getImagesByName(srv *Server, w http.ResponseWriter, r *http.Request, vars m return nil } +func postImagesGetCache(srv *Server, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + apiConfig := &ApiImageConfig{} + if err := json.NewDecoder(r.Body).Decode(apiConfig); err != nil { + return err + } + + image, err := srv.ImageGetCached(apiConfig.Id, apiConfig.Config) + if err != nil { + return err + } + apiId := &ApiId{Id: image.Id} + b, err := json.Marshal(apiId) + if err != nil { + return err + } + writeJson(w, b) + return nil +} + func ListenAndServe(addr string, srv *Server, logging bool) error { r := mux.NewRouter() log.Printf("Listening for HTTP on %s\n", addr) @@ -615,11 +621,11 @@ func ListenAndServe(addr string, srv *Server, logging bool) error { "POST": { "/auth": postAuth, "/commit": postCommit, - "/build": postBuild, "/images/create": postImagesCreate, "/images/{name:.*}/insert": postImagesInsert, "/images/{name:.*}/push": postImagesPush, "/images/{name:.*}/tag": postImagesTag, + "/images/getCache": postImagesGetCache, "/containers/create": postContainersCreate, "/containers/{name:.*}/kill": postContainersKill, "/containers/{name:.*}/restart": postContainersRestart, diff --git a/api_params.go b/api_params.go index e6f1c1b0b4..1a24ab2875 100644 --- a/api_params.go +++ b/api_params.go @@ -64,3 +64,8 @@ type ApiWait struct { type ApiAuth struct { Status string } + +type ApiImageConfig struct { + Id string + *Config +} diff --git a/builder.go b/builder.go index 1497644779..5f56f65d05 100644 --- a/builder.go +++ b/builder.go @@ -1,14 +1,9 @@ package docker import ( - "bufio" - "encoding/json" "fmt" - "github.com/dotcloud/docker/utils" - "io" "os" "path" - "strings" "time" ) @@ -16,6 +11,9 @@ type Builder struct { runtime *Runtime repositories *TagStore graph *Graph + + config *Config + image *Image } func NewBuilder(runtime *Runtime) *Builder { @@ -26,45 +24,6 @@ func NewBuilder(runtime *Runtime) *Builder { } } -func (builder *Builder) mergeConfig(userConf, imageConf *Config) { - if userConf.Hostname != "" { - userConf.Hostname = imageConf.Hostname - } - if userConf.User != "" { - userConf.User = imageConf.User - } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory - } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.CpuShares == 0 { - userConf.CpuShares = imageConf.CpuShares - } - if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { - userConf.PortSpecs = imageConf.PortSpecs - } - if !userConf.Tty { - userConf.Tty = imageConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } -} - func (builder *Builder) Create(config *Config) (*Container, error) { // Lookup image img, err := builder.repositories.LookupImage(config.Image) @@ -73,7 +32,7 @@ func (builder *Builder) Create(config *Config) (*Container, error) { } if img.Config != nil { - builder.mergeConfig(config, img.Config) + MergeConfig(config, img.Config) } if config.Cmd == nil || len(config.Cmd) == 0 { @@ -157,312 +116,3 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a } return img, nil } - -func (builder *Builder) clearTmp(containers, images map[string]struct{}) { - for c := range containers { - tmp := builder.runtime.Get(c) - builder.runtime.Destroy(tmp) - utils.Debugf("Removing container %s", c) - } - for i := range images { - builder.runtime.graph.Delete(i) - utils.Debugf("Removing image %s", i) - } -} - -func (builder *Builder) getCachedImage(image *Image, config *Config) (*Image, error) { - // Retrieve all images - images, err := builder.graph.All() - if err != nil { - return nil, err - } - - // Store the tree in a map of map (map[parentId][childId]) - imageMap := make(map[string]map[string]struct{}) - for _, img := range images { - if _, exists := imageMap[img.Parent]; !exists { - imageMap[img.Parent] = make(map[string]struct{}) - } - imageMap[img.Parent][img.Id] = struct{}{} - } - - // Loop on the children of the given image and check the config - for elem := range imageMap[image.Id] { - img, err := builder.graph.Get(elem) - if err != nil { - return nil, err - } - if CompareConfig(&img.ContainerConfig, config) { - return img, nil - } - } - return nil, nil -} - -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { - var ( - image, base *Image - config *Config - maintainer string - env map[string]string = make(map[string]string) - tmpContainers map[string]struct{} = make(map[string]struct{}) - tmpImages map[string]struct{} = make(map[string]struct{}) - ) - defer builder.clearTmp(tmpContainers, tmpImages) - - file := bufio.NewReader(dockerfile) - for { - line, err := file.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) - // Skip comments and empty line - if len(line) == 0 || line[0] == '#' { - continue - } - tmp := strings.SplitN(line, " ", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid Dockerfile format") - } - instruction := strings.Trim(tmp[0], " ") - arguments := strings.Trim(tmp[1], " ") - switch strings.ToLower(instruction) { - case "from": - fmt.Fprintf(stdout, "FROM %s\n", arguments) - image, err = builder.runtime.repositories.LookupImage(arguments) - if err != nil { - // if builder.runtime.graph.IsNotExist(err) { - - // var tag, remote string - // if strings.Contains(arguments, ":") { - // remoteParts := strings.Split(arguments, ":") - // tag = remoteParts[1] - // remote = remoteParts[0] - // } else { - // remote = arguments - // } - - // panic("TODO: reimplement this") - // // if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { - // // return nil, err - // // } - - // image, err = builder.runtime.repositories.LookupImage(arguments) - // if err != nil { - // return nil, err - // } - // } else { - return nil, err - // } - } - config = &Config{} - - break - case "maintainer": - fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) - maintainer = arguments - break - case "run": - fmt.Fprintf(stdout, "RUN %s\n", arguments) - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to run") - } - config, _, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, builder.runtime.capabilities) - if err != nil { - return nil, err - } - - for key, value := range env { - config.Env = append(config.Env, fmt.Sprintf("%s=%s", key, value)) - } - - if cache, err := builder.getCachedImage(image, config); err != nil { - return nil, err - } else if cache != nil { - image = cache - fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) - break - } - - utils.Debugf("Env -----> %v ------ %v\n", config.Env, env) - - // Create the container and start it - c, err := builder.Create(config) - if err != nil { - return nil, err - } - - if os.Getenv("DEBUG") != "" { - out, _ := c.StdoutPipe() - err2, _ := c.StderrPipe() - go io.Copy(os.Stdout, out) - go io.Copy(os.Stdout, err2) - } - - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - // Wait for it to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - // use the base as the new image - image = base - - break - case "env": - tmp := strings.SplitN(arguments, " ", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid ENV format") - } - key := strings.Trim(tmp[0], " ") - value := strings.Trim(tmp[1], " ") - fmt.Fprintf(stdout, "ENV %s %s\n", key, value) - env[key] = value - if image != nil { - fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) - } else { - fmt.Fprintf(stdout, "===> \n") - } - break - case "cmd": - fmt.Fprintf(stdout, "CMD %s\n", arguments) - - // Create the container and start it - c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - cmd := []string{} - if err := json.Unmarshal([]byte(arguments), &cmd); err != nil { - return nil, err - } - config.Cmd = cmd - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, config) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - image = base - break - case "expose": - ports := strings.Split(arguments, " ") - - fmt.Fprintf(stdout, "EXPOSE %v\n", ports) - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - - // Create the container and start it - c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - config.PortSpecs = append(ports, config.PortSpecs...) - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, config) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - image = base - break - case "insert": - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - tmp = strings.SplitN(arguments, " ", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid INSERT format") - } - sourceUrl := strings.Trim(tmp[0], " ") - destPath := strings.Trim(tmp[1], " ") - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - - file, err := utils.Download(sourceUrl, stdout) - if err != nil { - return nil, err - } - defer file.Body.Close() - - config, _, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, builder.runtime.capabilities) - if err != nil { - return nil, err - } - c, err := builder.Create(config) - if err != nil { - return nil, err - } - - if err := c.Start(); err != nil { - return nil, err - } - - // Wait for echo to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - if err := c.Inject(file.Body, destPath); err != nil { - return nil, err - } - - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - image = base - - break - default: - fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) - } - } - if image != nil { - // The build is successful, keep the temporary containers and images - for i := range tmpImages { - delete(tmpImages, i) - } - for i := range tmpContainers { - delete(tmpContainers, i) - } - fmt.Fprintf(stdout, "Build finished. image id: %s\n", image.ShortId()) - return image, nil - } - return nil, fmt.Errorf("An error occured during the build\n") -} diff --git a/builder_client.go b/builder_client.go new file mode 100644 index 0000000000..4a29129ed2 --- /dev/null +++ b/builder_client.go @@ -0,0 +1,275 @@ +package docker + +import ( + "bufio" + "encoding/json" + "fmt" + "github.com/dotcloud/docker/utils" + "io" + "net/url" + "os" + "reflect" + "strings" +) + +type BuilderClient struct { + builder *Builder + cli *DockerCli + + image string + maintainer string + config *Config + + tmpContainers map[string]struct{} + tmpImages map[string]struct{} + + needCommit bool +} + +func (b *BuilderClient) clearTmp(containers, images map[string]struct{}) { + for c := range containers { + tmp := b.builder.runtime.Get(c) + b.builder.runtime.Destroy(tmp) + utils.Debugf("Removing container %s", c) + } + for i := range images { + b.builder.runtime.graph.Delete(i) + utils.Debugf("Removing image %s", i) + } +} + +func (b *BuilderClient) From(name string) error { + obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) + if statusCode == 404 { + if err := b.cli.hijack("POST", "/images/create?fromImage="+name, false); err != nil { + return err + } + obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil) + if err != nil { + return err + } + } + if err != nil { + return err + } + + img := &ApiImages{} + if err := json.Unmarshal(obj, img); err != nil { + return err + } + b.image = img.Id + return nil +} + +func (b *BuilderClient) Maintainer(name string) error { + b.needCommit = true + b.maintainer = name + return nil +} + +func (b *BuilderClient) Run(args string) error { + if b.image == "" { + return fmt.Errorf("Please provide a source image with `from` prior to run") + } + config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, b.builder.runtime.capabilities) + if err != nil { + return err + } + MergeConfig(b.config, config) + body, statusCode, err := b.cli.call("POST", "/images/getCache", &ApiImageConfig{Id: b.image, Config: b.config}) + if err != nil { + if statusCode != 404 { + return err + } + } + if statusCode != 404 { + apiId := &ApiId{} + if err := json.Unmarshal(body, apiId); err != nil { + return err + } + b.image = apiId.Id + return nil + } + + body, _, err = b.cli.call("POST", "/containers/create", b.config) + if err != nil { + return err + } + + out := &ApiRun{} + err = json.Unmarshal(body, out) + if err != nil { + return err + } + + for _, warning := range out.Warnings { + fmt.Fprintln(os.Stderr, "WARNING: ", warning) + } + + //start the container + _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/start", nil) + if err != nil { + return err + } + b.tmpContainers[out.Id] = struct{}{} + + // Wait for it to finish + _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/wait", nil) + if err != nil { + return err + } + + // Commit the container + v := url.Values{} + v.Set("container", out.Id) + v.Set("author", b.maintainer) + body, _, err = b.cli.call("POST", "/commit?"+v.Encode(), b.config) + if err != nil { + return err + } + apiId := &ApiId{} + err = json.Unmarshal(body, apiId) + if err != nil { + return err + } + b.tmpImages[apiId.Id] = struct{}{} + b.image = apiId.Id + b.needCommit = false + return nil +} + +func (b *BuilderClient) Env(args string) error { + b.needCommit = true + tmp := strings.SplitN(args, " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid ENV format") + } + key := strings.Trim(tmp[0], " ") + value := strings.Trim(tmp[1], " ") + + for i, elem := range b.config.Env { + if strings.HasPrefix(elem, key+"=") { + b.config.Env[i] = key + "=" + value + return nil + } + } + b.config.Env = append(b.config.Env, key+"="+value) + return nil +} + +func (b *BuilderClient) Cmd(args string) error { + b.needCommit = true + b.config.Cmd = []string{"/bin/sh", "-c", args} + return nil +} + +func (b *BuilderClient) Expose(args string) error { + ports := strings.Split(args, " ") + b.config.PortSpecs = append(ports, b.config.PortSpecs...) + return nil +} + +func (b *BuilderClient) Insert(args string) error { + // FIXME: Reimplement this once the remove_hijack branch gets merged. + // We need to retrieve the resulting Id + return fmt.Errorf("INSERT not implemented") +} + +func NewBuilderClient(dockerfile io.Reader) (string, error) { + // defer b.clearTmp(tmpContainers, tmpImages) + + b := &BuilderClient{ + cli: NewDockerCli("0.0.0.0", 4243), + } + file := bufio.NewReader(dockerfile) + for { + line, err := file.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return "", err + } + line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) + // Skip comments and empty line + if len(line) == 0 || line[0] == '#' { + continue + } + tmp := strings.SplitN(line, " ", 2) + if len(tmp) != 2 { + return "", fmt.Errorf("Invalid Dockerfile format") + } + instruction := strings.ToLower(strings.Trim(tmp[0], " ")) + arguments := strings.Trim(tmp[1], " ") + + fmt.Printf("%s %s\n", strings.ToUpper(instruction), arguments) + + method, exists := reflect.TypeOf(b).MethodByName(strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) + if !exists { + fmt.Printf("Skipping unknown instruction %s\n", strings.ToUpper(instruction)) + } + ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() + if ret != nil { + return "", ret.(error) + } + + fmt.Printf("===> %v\n", b.image) + } + if b.needCommit { + body, _, err = b.cli.call("POST", "/containers/create", b.config) + if err != nil { + return err + } + + out := &ApiRun{} + err = json.Unmarshal(body, out) + if err != nil { + return err + } + + for _, warning := range out.Warnings { + fmt.Fprintln(os.Stderr, "WARNING: ", warning) + } + + //start the container + _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/start", nil) + if err != nil { + return err + } + b.tmpContainers[out.Id] = struct{}{} + + // Wait for it to finish + _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/wait", nil) + if err != nil { + return err + } + + // Commit the container + v := url.Values{} + v.Set("container", out.Id) + v.Set("author", b.maintainer) + body, _, err = b.cli.call("POST", "/commit?"+v.Encode(), b.config) + if err != nil { + return err + } + apiId := &ApiId{} + err = json.Unmarshal(body, apiId) + if err != nil { + return err + } + b.tmpImages[apiId.Id] = struct{}{} + b.image = apiId.Id + } + if b.image != "" { + // The build is successful, keep the temporary containers and images + for i := range b.tmpImages { + delete(b.tmpImages, i) + } + for i := range b.tmpContainers { + delete(b.tmpContainers, i) + } + fmt.Printf("Build finished. image id: %s\n", b.image) + return b.image, nil + } + return "", fmt.Errorf("An error occured during the build\n") +} diff --git a/commands.go b/commands.go index 33ba8125de..4d63782f3d 100644 --- a/commands.go +++ b/commands.go @@ -54,37 +54,37 @@ func ParseCommands(args ...string) error { func (cli *DockerCli) CmdHelp(args ...string) error { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" - for _, cmd := range [][]string{ - {"attach", "Attach to a running container"}, - {"build", "Build a container from Dockerfile via stdin"}, - {"commit", "Create a new image from a container's changes"}, - {"diff", "Inspect changes on a container's filesystem"}, - {"export", "Stream the contents of a container as a tar archive"}, - {"history", "Show the history of an image"}, - {"images", "List images"}, - {"import", "Create a new filesystem image from the contents of a tarball"}, - {"info", "Display system-wide information"}, - {"insert", "Insert a file in an image"}, - {"inspect", "Return low-level information on a container"}, - {"kill", "Kill a running container"}, - {"login", "Register or Login to the docker registry server"}, - {"logs", "Fetch the logs of a container"}, - {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, - {"ps", "List containers"}, - {"pull", "Pull an image or a repository from the docker registry server"}, - {"push", "Push an image or a repository to the docker registry server"}, - {"restart", "Restart a running container"}, - {"rm", "Remove a container"}, - {"rmi", "Remove an image"}, - {"run", "Run a command in a new container"}, - {"search", "Search for an image in the docker index"}, - {"start", "Start a stopped container"}, - {"stop", "Stop a running container"}, - {"tag", "Tag an image into a repository"}, - {"version", "Show the docker version information"}, - {"wait", "Block until a container stops, then print its exit code"}, + for cmd, description := range map[string]string{ + "attach": "Attach to a running container", + "build": "Build a container from Dockerfile or via stdin", + "commit": "Create a new image from a container's changes", + "diff": "Inspect changes on a container's filesystem", + "export": "Stream the contents of a container as a tar archive", + "history": "Show the history of an image", + "images": "List images", + "import": "Create a new filesystem image from the contents of a tarball", + "info": "Display system-wide information", + "insert": "Insert a file in an image", + "inspect": "Return low-level information on a container", + "kill": "Kill a running container", + "login": "Register or Login to the docker registry server", + "logs": "Fetch the logs of a container", + "port": "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT", + "ps": "List containers", + "pull": "Pull an image or a repository from the docker registry server", + "push": "Push an image or a repository to the docker registry server", + "restart": "Restart a running container", + "rm": "Remove a container", + "rmi": "Remove an image", + "run": "Run a command in a new container", + "search": "Search for an image in the docker index", + "start": "Start a stopped container", + "stop": "Stop a running container", + "tag": "Tag an image into a repository", + "version": "Show the docker version information", + "wait": "Block until a container stops, then print its exit code", } { - help += fmt.Sprintf(" %-10.10s%s\n", cmd[0], cmd[1]) + help += fmt.Sprintf(" %-10.10s%s\n", cmd, description) } fmt.Println(help) return nil @@ -112,15 +112,29 @@ func (cli *DockerCli) CmdInsert(args ...string) error { } func (cli *DockerCli) CmdBuild(args ...string) error { - cmd := Subcmd("build", "-", "Build an image from Dockerfile via stdin") + cmd := Subcmd("build", "-|Dockerfile", "Build an image from Dockerfile or via stdin") if err := cmd.Parse(args); err != nil { return nil } + var ( + file io.ReadCloser + err error + ) - err := cli.hijack("POST", "/build", false) - if err != nil { - return err + if cmd.NArg() == 0 { + file, err = os.Open("Dockerfile") + if err != nil { + return err + } + } else if cmd.Arg(0) == "-" { + file = os.Stdin + } else { + file, err = os.Open(cmd.Arg(0)) + if err != nil { + return err + } } + NewBuilderClient(file) return nil } diff --git a/server.go b/server.go index dafa44ec84..9877b1efdd 100644 --- a/server.go +++ b/server.go @@ -140,8 +140,10 @@ func (srv *Server) ImagesViz(out io.Writer) error { } func (srv *Server) Images(all bool, filter string) ([]ApiImages, error) { - var allImages map[string]*Image - var err error + var ( + allImages map[string]*Image + err error + ) if all { allImages, err = srv.runtime.graph.Map() } else { @@ -150,7 +152,7 @@ func (srv *Server) Images(all bool, filter string) ([]ApiImages, error) { if err != nil { return nil, err } - var outs []ApiImages = []ApiImages{} //produce [] when empty instead of 'null' + outs := []ApiImages{} //produce [] when empty instead of 'null' for name, repository := range srv.runtime.repositories.Repositories { if filter != "" && name != filter { continue @@ -653,15 +655,6 @@ func (srv *Server) ContainerCreate(config *Config) (string, error) { return container.ShortId(), nil } -func (srv *Server) ImageCreateFromFile(dockerfile io.Reader, out io.Writer) error { - img, err := NewBuilder(srv.runtime).Build(dockerfile, out) - if err != nil { - return err - } - fmt.Fprintf(out, "%s\n", img.ShortId()) - return nil -} - func (srv *Server) ContainerRestart(name string, t int) error { if container := srv.runtime.Get(name); container != nil { if err := container.Restart(t); err != nil { @@ -722,6 +715,36 @@ func (srv *Server) ImageDelete(name string) error { return nil } +func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) { + + // Retrieve all images + images, err := srv.runtime.graph.All() + if err != nil { + return nil, err + } + + // Store the tree in a map of map (map[parentId][childId]) + imageMap := make(map[string]map[string]struct{}) + for _, img := range images { + if _, exists := imageMap[img.Parent]; !exists { + imageMap[img.Parent] = make(map[string]struct{}) + } + imageMap[img.Parent][img.Id] = struct{}{} + } + + // Loop on the children of the given image and check the config + for elem := range imageMap[imgId] { + img, err := srv.runtime.graph.Get(elem) + if err != nil { + return nil, err + } + if CompareConfig(&img.ContainerConfig, config) { + return img, nil + } + } + return nil, nil +} + func (srv *Server) ContainerStart(name string) error { if container := srv.runtime.Get(name); container != nil { if err := container.Start(); err != nil { diff --git a/utils.go b/utils.go index d67f50e526..27478002d3 100644 --- a/utils.go +++ b/utils.go @@ -47,3 +47,42 @@ func CompareConfig(a, b *Config) bool { return true } + +func MergeConfig(userConf, imageConf *Config) { + if userConf.Hostname != "" { + userConf.Hostname = imageConf.Hostname + } + if userConf.User != "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.CpuShares == 0 { + userConf.CpuShares = imageConf.CpuShares + } + if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { + userConf.PortSpecs = imageConf.PortSpecs + } + if !userConf.Tty { + userConf.Tty = imageConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } +} From c2a14bb196d0d3046e185783ebacd4b83fa36dd4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 12:09:15 -0700 Subject: [PATCH 02/12] Add "Cmd" prefix to builder instructions --- api.go | 4 ++ builder_client.go | 164 +++++++++++++++++++--------------------------- commands.go | 4 +- 3 files changed, 73 insertions(+), 99 deletions(-) diff --git a/api.go b/api.go index 5cb3da85c2..ecad0b1f4a 100644 --- a/api.go +++ b/api.go @@ -590,6 +590,10 @@ func postImagesGetCache(srv *Server, w http.ResponseWriter, r *http.Request, var if err != nil { return err } + if image == nil { + w.WriteHeader(http.StatusNotFound) + return nil + } apiId := &ApiId{Id: image.Id} b, err := json.Marshal(apiId) if err != nil { diff --git a/builder_client.go b/builder_client.go index 4a29129ed2..0ad35045dd 100644 --- a/builder_client.go +++ b/builder_client.go @@ -13,8 +13,7 @@ import ( ) type BuilderClient struct { - builder *Builder - cli *DockerCli + cli *DockerCli image string maintainer string @@ -28,17 +27,20 @@ type BuilderClient struct { func (b *BuilderClient) clearTmp(containers, images map[string]struct{}) { for c := range containers { - tmp := b.builder.runtime.Get(c) - b.builder.runtime.Destroy(tmp) + if _, _, err := b.cli.call("DELETE", "/containers/"+c, nil); err != nil { + utils.Debugf("%s", err) + } utils.Debugf("Removing container %s", c) } for i := range images { - b.builder.runtime.graph.Delete(i) + if _, _, err := b.cli.call("DELETE", "/images/"+i, nil); err != nil { + utils.Debugf("%s", err) + } utils.Debugf("Removing image %s", i) } } -func (b *BuilderClient) From(name string) error { +func (b *BuilderClient) CmdFrom(name string) error { obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) if statusCode == 404 { if err := b.cli.hijack("POST", "/images/create?fromImage="+name, false); err != nil { @@ -53,7 +55,7 @@ func (b *BuilderClient) From(name string) error { return err } - img := &ApiImages{} + img := &ApiId{} if err := json.Unmarshal(obj, img); err != nil { return err } @@ -61,17 +63,17 @@ func (b *BuilderClient) From(name string) error { return nil } -func (b *BuilderClient) Maintainer(name string) error { +func (b *BuilderClient) CmdMaintainer(name string) error { b.needCommit = true b.maintainer = name return nil } -func (b *BuilderClient) Run(args string) error { +func (b *BuilderClient) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } - config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, b.builder.runtime.capabilities) + config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil) if err != nil { return err } @@ -90,8 +92,49 @@ func (b *BuilderClient) Run(args string) error { b.image = apiId.Id return nil } + b.commit() + return nil +} - body, _, err = b.cli.call("POST", "/containers/create", b.config) +func (b *BuilderClient) CmdEnv(args string) error { + b.needCommit = true + tmp := strings.SplitN(args, " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid ENV format") + } + key := strings.Trim(tmp[0], " ") + value := strings.Trim(tmp[1], " ") + + for i, elem := range b.config.Env { + if strings.HasPrefix(elem, key+"=") { + b.config.Env[i] = key + "=" + value + return nil + } + } + b.config.Env = append(b.config.Env, key+"="+value) + return nil +} + +func (b *BuilderClient) CmdCmd(args string) error { + b.needCommit = true + b.config.Cmd = []string{"/bin/sh", "-c", args} + return nil +} + +func (b *BuilderClient) CmdExpose(args string) error { + ports := strings.Split(args, " ") + b.config.PortSpecs = append(ports, b.config.PortSpecs...) + return nil +} + +func (b *BuilderClient) CmdInsert(args string) error { + // FIXME: Reimplement this once the remove_hijack branch gets merged. + // We need to retrieve the resulting Id + return fmt.Errorf("INSERT not implemented") +} + +func (b *BuilderClient) commit() error { + body, _, err := b.cli.call("POST", "/containers/create", b.config) if err != nil { return err } @@ -134,53 +177,11 @@ func (b *BuilderClient) Run(args string) error { } b.tmpImages[apiId.Id] = struct{}{} b.image = apiId.Id - b.needCommit = false return nil } -func (b *BuilderClient) Env(args string) error { - b.needCommit = true - tmp := strings.SplitN(args, " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid ENV format") - } - key := strings.Trim(tmp[0], " ") - value := strings.Trim(tmp[1], " ") - - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } - } - b.config.Env = append(b.config.Env, key+"="+value) - return nil -} - -func (b *BuilderClient) Cmd(args string) error { - b.needCommit = true - b.config.Cmd = []string{"/bin/sh", "-c", args} - return nil -} - -func (b *BuilderClient) Expose(args string) error { - ports := strings.Split(args, " ") - b.config.PortSpecs = append(ports, b.config.PortSpecs...) - return nil -} - -func (b *BuilderClient) Insert(args string) error { - // FIXME: Reimplement this once the remove_hijack branch gets merged. - // We need to retrieve the resulting Id - return fmt.Errorf("INSERT not implemented") -} - -func NewBuilderClient(dockerfile io.Reader) (string, error) { +func (b *BuilderClient) Build(dockerfile io.Reader) (string, error) { // defer b.clearTmp(tmpContainers, tmpImages) - - b := &BuilderClient{ - cli: NewDockerCli("0.0.0.0", 4243), - } file := bufio.NewReader(dockerfile) for { line, err := file.ReadString('\n') @@ -204,7 +205,7 @@ func NewBuilderClient(dockerfile io.Reader) (string, error) { fmt.Printf("%s %s\n", strings.ToUpper(instruction), arguments) - method, exists := reflect.TypeOf(b).MethodByName(strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) + method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) if !exists { fmt.Printf("Skipping unknown instruction %s\n", strings.ToUpper(instruction)) } @@ -216,49 +217,7 @@ func NewBuilderClient(dockerfile io.Reader) (string, error) { fmt.Printf("===> %v\n", b.image) } if b.needCommit { - body, _, err = b.cli.call("POST", "/containers/create", b.config) - if err != nil { - return err - } - - out := &ApiRun{} - err = json.Unmarshal(body, out) - if err != nil { - return err - } - - for _, warning := range out.Warnings { - fmt.Fprintln(os.Stderr, "WARNING: ", warning) - } - - //start the container - _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/start", nil) - if err != nil { - return err - } - b.tmpContainers[out.Id] = struct{}{} - - // Wait for it to finish - _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/wait", nil) - if err != nil { - return err - } - - // Commit the container - v := url.Values{} - v.Set("container", out.Id) - v.Set("author", b.maintainer) - body, _, err = b.cli.call("POST", "/commit?"+v.Encode(), b.config) - if err != nil { - return err - } - apiId := &ApiId{} - err = json.Unmarshal(body, apiId) - if err != nil { - return err - } - b.tmpImages[apiId.Id] = struct{}{} - b.image = apiId.Id + b.commit() } if b.image != "" { // The build is successful, keep the temporary containers and images @@ -273,3 +232,12 @@ func NewBuilderClient(dockerfile io.Reader) (string, error) { } return "", fmt.Errorf("An error occured during the build\n") } + +func NewBuilderClient(addr string, port int) *BuilderClient { + return &BuilderClient{ + cli: NewDockerCli(addr, port), + config: &Config{}, + tmpContainers: make(map[string]struct{}), + tmpImages: make(map[string]struct{}), + } +} diff --git a/commands.go b/commands.go index 4d63782f3d..8da400abe8 100644 --- a/commands.go +++ b/commands.go @@ -134,7 +134,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { return err } } - NewBuilderClient(file) + if _, err := NewBuilderClient("0.0.0.0", 4243).Build(file); err != nil { + return err + } return nil } From b51303cddce2cbb22fc3ee0de9d9125bec2f029a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 13:50:50 -0700 Subject: [PATCH 03/12] Make sure to have a command to execute upon commit --- builder_client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder_client.go b/builder_client.go index 0ad35045dd..0c7ea15e13 100644 --- a/builder_client.go +++ b/builder_client.go @@ -134,6 +134,10 @@ func (b *BuilderClient) CmdInsert(args string) error { } func (b *BuilderClient) commit() error { + if b.config.Cmd == nil || len(b.config.Cmd) < 1 { + b.config.Cmd = []string{"echo"} + } + body, _, err := b.cli.call("POST", "/containers/create", b.config) if err != nil { return err From c6bc90d02daa2f7086a1c7469c7239911c2bc357 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 15:02:32 -0700 Subject: [PATCH 04/12] Isolate run() from commit --- builder_client.go | 83 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 23 deletions(-) diff --git a/builder_client.go b/builder_client.go index 0c7ea15e13..660ef12f8c 100644 --- a/builder_client.go +++ b/builder_client.go @@ -77,7 +77,11 @@ func (b *BuilderClient) CmdRun(args string) error { if err != nil { return err } + + cmd, env := b.config.Cmd, b.config.Env + b.config.Cmd = nil MergeConfig(b.config, config) + body, statusCode, err := b.cli.call("POST", "/images/getCache", &ApiImageConfig{Id: b.image, Config: b.config}) if err != nil { if statusCode != 404 { @@ -89,11 +93,16 @@ func (b *BuilderClient) CmdRun(args string) error { if err := json.Unmarshal(body, apiId); err != nil { return err } + utils.Debugf("Use cached version") b.image = apiId.Id return nil } - b.commit() - return nil + cid, err := b.run() + if err != nil { + return err + } + b.config.Cmd, b.config.Env = cmd, env + return b.commit(cid) } func (b *BuilderClient) CmdEnv(args string) error { @@ -133,54 +142,80 @@ func (b *BuilderClient) CmdInsert(args string) error { return fmt.Errorf("INSERT not implemented") } -func (b *BuilderClient) commit() error { - if b.config.Cmd == nil || len(b.config.Cmd) < 1 { - b.config.Cmd = []string{"echo"} +func (b *BuilderClient) run() (string, error) { + if b.image == "" { + return "", fmt.Errorf("Please provide a source image with `from` prior to run") } - + b.config.Image = b.image body, _, err := b.cli.call("POST", "/containers/create", b.config) if err != nil { - return err + return "", err } - out := &ApiRun{} - err = json.Unmarshal(body, out) - if err != nil { - return err + apiRun := &ApiRun{} + if err := json.Unmarshal(body, apiRun); err != nil { + return "", err } - - for _, warning := range out.Warnings { + for _, warning := range apiRun.Warnings { fmt.Fprintln(os.Stderr, "WARNING: ", warning) } //start the container - _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/start", nil) + _, _, err = b.cli.call("POST", "/containers/"+apiRun.Id+"/start", nil) if err != nil { - return err + return "", err } - b.tmpContainers[out.Id] = struct{}{} + b.tmpContainers[apiRun.Id] = struct{}{} // Wait for it to finish - _, _, err = b.cli.call("POST", "/containers/"+out.Id+"/wait", nil) + body, _, err = b.cli.call("POST", "/containers/"+apiRun.Id+"/wait", nil) if err != nil { - return err + return "", err + } + apiWait := &ApiWait{} + if err := json.Unmarshal(body, apiWait); err != nil { + return "", err + } + if apiWait.StatusCode != 0 { + return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, apiWait.StatusCode) + } + + return apiRun.Id, nil +} + +func (b *BuilderClient) commit(id string) error { + if b.image == "" { + return fmt.Errorf("Please provide a source image with `from` prior to run") + } + b.config.Image = b.image + + if id == "" { + cmd := b.config.Cmd + b.config.Cmd = []string{"true"} + if cid, err := b.run(); err != nil { + return err + } else { + id = cid + } + b.config.Cmd = cmd } // Commit the container v := url.Values{} - v.Set("container", out.Id) + v.Set("container", id) v.Set("author", b.maintainer) - body, _, err = b.cli.call("POST", "/commit?"+v.Encode(), b.config) + + body, _, err := b.cli.call("POST", "/commit?"+v.Encode(), b.config) if err != nil { return err } apiId := &ApiId{} - err = json.Unmarshal(body, apiId) - if err != nil { + if err := json.Unmarshal(body, apiId); err != nil { return err } b.tmpImages[apiId.Id] = struct{}{} b.image = apiId.Id + b.needCommit = false return nil } @@ -221,7 +256,9 @@ func (b *BuilderClient) Build(dockerfile io.Reader) (string, error) { fmt.Printf("===> %v\n", b.image) } if b.needCommit { - b.commit() + if err := b.commit(""); err != nil { + return "", err + } } if b.image != "" { // The build is successful, keep the temporary containers and images From b06784b0dd2e86a7450fab30707468acfae5d7aa Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 16:00:16 -0700 Subject: [PATCH 05/12] Make docker client an interface --- builder_client.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/builder_client.go b/builder_client.go index 660ef12f8c..5000782f81 100644 --- a/builder_client.go +++ b/builder_client.go @@ -12,7 +12,13 @@ import ( "strings" ) -type BuilderClient struct { +type BuilderClient interface { + Build(io.Reader) (string, error) + CmdFrom(string) error + CmdRun(string) error +} + +type builderClient struct { cli *DockerCli image string @@ -25,7 +31,7 @@ type BuilderClient struct { needCommit bool } -func (b *BuilderClient) clearTmp(containers, images map[string]struct{}) { +func (b builderClient) clearTmp(containers, images map[string]struct{}) { for c := range containers { if _, _, err := b.cli.call("DELETE", "/containers/"+c, nil); err != nil { utils.Debugf("%s", err) @@ -40,7 +46,7 @@ func (b *BuilderClient) clearTmp(containers, images map[string]struct{}) { } } -func (b *BuilderClient) CmdFrom(name string) error { +func (b builderClient) CmdFrom(name string) error { obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) if statusCode == 404 { if err := b.cli.hijack("POST", "/images/create?fromImage="+name, false); err != nil { @@ -63,13 +69,13 @@ func (b *BuilderClient) CmdFrom(name string) error { return nil } -func (b *BuilderClient) CmdMaintainer(name string) error { +func (b builderClient) CmdMaintainer(name string) error { b.needCommit = true b.maintainer = name return nil } -func (b *BuilderClient) CmdRun(args string) error { +func (b builderClient) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -105,7 +111,7 @@ func (b *BuilderClient) CmdRun(args string) error { return b.commit(cid) } -func (b *BuilderClient) CmdEnv(args string) error { +func (b builderClient) CmdEnv(args string) error { b.needCommit = true tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -124,25 +130,25 @@ func (b *BuilderClient) CmdEnv(args string) error { return nil } -func (b *BuilderClient) CmdCmd(args string) error { +func (b builderClient) CmdCmd(args string) error { b.needCommit = true b.config.Cmd = []string{"/bin/sh", "-c", args} return nil } -func (b *BuilderClient) CmdExpose(args string) error { +func (b builderClient) CmdExpose(args string) error { ports := strings.Split(args, " ") b.config.PortSpecs = append(ports, b.config.PortSpecs...) return nil } -func (b *BuilderClient) CmdInsert(args string) error { +func (b builderClient) CmdInsert(args string) error { // FIXME: Reimplement this once the remove_hijack branch gets merged. // We need to retrieve the resulting Id return fmt.Errorf("INSERT not implemented") } -func (b *BuilderClient) run() (string, error) { +func (b builderClient) run() (string, error) { if b.image == "" { return "", fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -183,7 +189,7 @@ func (b *BuilderClient) run() (string, error) { return apiRun.Id, nil } -func (b *BuilderClient) commit(id string) error { +func (b builderClient) commit(id string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -219,7 +225,7 @@ func (b *BuilderClient) commit(id string) error { return nil } -func (b *BuilderClient) Build(dockerfile io.Reader) (string, error) { +func (b builderClient) Build(dockerfile io.Reader) (string, error) { // defer b.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -274,8 +280,8 @@ func (b *BuilderClient) Build(dockerfile io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuilderClient(addr string, port int) *BuilderClient { - return &BuilderClient{ +func NewBuilderClient(addr string, port int) BuilderClient { + return &builderClient{ cli: NewDockerCli(addr, port), config: &Config{}, tmpContainers: make(map[string]struct{}), From 13e687e5790f05552f9be84bf1d60d37fca4c078 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 16:00:51 -0700 Subject: [PATCH 06/12] Allow multiple syntaxes for CMD --- builder_client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/builder_client.go b/builder_client.go index 5000782f81..0c283d2591 100644 --- a/builder_client.go +++ b/builder_client.go @@ -132,7 +132,12 @@ func (b builderClient) CmdEnv(args string) error { func (b builderClient) CmdCmd(args string) error { b.needCommit = true - b.config.Cmd = []string{"/bin/sh", "-c", args} + var cmd []string + if err := json.Unmarshal([]byte(args), &cmd); err != nil { + b.config.Cmd = []string{"/bin/sh", "-c", args} + } else { + b.config.Cmd = cmd + } return nil } From f35f084059e5c34940f74f55ad32ecfcd78ce61c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 16:35:28 -0700 Subject: [PATCH 07/12] Use pointers for the object methods --- builder_client.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/builder_client.go b/builder_client.go index 0c283d2591..45e191ea3d 100644 --- a/builder_client.go +++ b/builder_client.go @@ -31,7 +31,7 @@ type builderClient struct { needCommit bool } -func (b builderClient) clearTmp(containers, images map[string]struct{}) { +func (b *builderClient) clearTmp(containers, images map[string]struct{}) { for c := range containers { if _, _, err := b.cli.call("DELETE", "/containers/"+c, nil); err != nil { utils.Debugf("%s", err) @@ -46,7 +46,7 @@ func (b builderClient) clearTmp(containers, images map[string]struct{}) { } } -func (b builderClient) CmdFrom(name string) error { +func (b *builderClient) CmdFrom(name string) error { obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) if statusCode == 404 { if err := b.cli.hijack("POST", "/images/create?fromImage="+name, false); err != nil { @@ -66,16 +66,17 @@ func (b builderClient) CmdFrom(name string) error { return err } b.image = img.Id + utils.Debugf("Using image %s", b.image) return nil } -func (b builderClient) CmdMaintainer(name string) error { +func (b *builderClient) CmdMaintainer(name string) error { b.needCommit = true b.maintainer = name return nil } -func (b builderClient) CmdRun(args string) error { +func (b *builderClient) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -111,7 +112,7 @@ func (b builderClient) CmdRun(args string) error { return b.commit(cid) } -func (b builderClient) CmdEnv(args string) error { +func (b *builderClient) CmdEnv(args string) error { b.needCommit = true tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -130,10 +131,11 @@ func (b builderClient) CmdEnv(args string) error { return nil } -func (b builderClient) CmdCmd(args string) error { +func (b *builderClient) CmdCmd(args string) error { b.needCommit = true var cmd []string if err := json.Unmarshal([]byte(args), &cmd); err != nil { + utils.Debugf("Error unmarshalling: %s, using /bin/sh -c", err) b.config.Cmd = []string{"/bin/sh", "-c", args} } else { b.config.Cmd = cmd @@ -141,19 +143,19 @@ func (b builderClient) CmdCmd(args string) error { return nil } -func (b builderClient) CmdExpose(args string) error { +func (b *builderClient) CmdExpose(args string) error { ports := strings.Split(args, " ") b.config.PortSpecs = append(ports, b.config.PortSpecs...) return nil } -func (b builderClient) CmdInsert(args string) error { +func (b *builderClient) CmdInsert(args string) error { // FIXME: Reimplement this once the remove_hijack branch gets merged. // We need to retrieve the resulting Id return fmt.Errorf("INSERT not implemented") } -func (b builderClient) run() (string, error) { +func (b *builderClient) run() (string, error) { if b.image == "" { return "", fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -194,7 +196,7 @@ func (b builderClient) run() (string, error) { return apiRun.Id, nil } -func (b builderClient) commit(id string) error { +func (b *builderClient) commit(id string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } @@ -230,7 +232,7 @@ func (b builderClient) commit(id string) error { return nil } -func (b builderClient) Build(dockerfile io.Reader) (string, error) { +func (b *builderClient) Build(dockerfile io.Reader) (string, error) { // defer b.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -253,7 +255,7 @@ func (b builderClient) Build(dockerfile io.Reader) (string, error) { instruction := strings.ToLower(strings.Trim(tmp[0], " ")) arguments := strings.Trim(tmp[1], " ") - fmt.Printf("%s %s\n", strings.ToUpper(instruction), arguments) + fmt.Printf("%s %s (%s)\n", strings.ToUpper(instruction), arguments, b.image) method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) if !exists { From 49505c599b3a65196b6b6f746f4bfad3a417dd7a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 17:30:33 -0700 Subject: [PATCH 08/12] Fix an issue trying to pull specific tag --- builder_client.go | 18 ++++++++++++++++-- server.go | 9 ++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/builder_client.go b/builder_client.go index 45e191ea3d..ceeab002c9 100644 --- a/builder_client.go +++ b/builder_client.go @@ -49,7 +49,21 @@ func (b *builderClient) clearTmp(containers, images map[string]struct{}) { func (b *builderClient) CmdFrom(name string) error { obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil) if statusCode == 404 { - if err := b.cli.hijack("POST", "/images/create?fromImage="+name, false); err != nil { + + remote := name + var tag string + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = remoteParts[1] + remote = remoteParts[0] + } + var out io.Writer + if os.Getenv("DEBUG") != "" { + out = os.Stdout + } else { + out = &utils.NopWriter{} + } + if err := b.cli.stream("POST", "/images/create?fromImage="+remote+"&tag="+tag, nil, out); err != nil { return err } obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil) @@ -233,7 +247,7 @@ func (b *builderClient) commit(id string) error { } func (b *builderClient) Build(dockerfile io.Reader) (string, error) { - // defer b.clearTmp(tmpContainers, tmpImages) + defer b.clearTmp(b.tmpContainers, b.tmpImages) file := bufio.NewReader(dockerfile) for { line, err := file.ReadString('\n') diff --git a/server.go b/server.go index e9cc44de6f..564b1c812d 100644 --- a/server.go +++ b/server.go @@ -363,7 +363,7 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string) error for _, img := range repoData.ImgList { if askedTag != "" && img.Tag != askedTag { - utils.Debugf("%s does not match %s, skipping", img.Tag, askedTag) + utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.Id) continue } fmt.Fprintf(out, "Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote) @@ -380,11 +380,10 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string) error return fmt.Errorf("Could not find repository on any of the indexed registries.") } } - // If we asked for a specific tag, do not register the others - if askedTag != "" { - return nil - } for tag, id := range tagsList { + if askedTag != "" && tag != askedTag { + continue + } if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil { return err } From 218812eb3cfeb5c5253ed6a54ed3e45c1107ffd1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 17:52:39 -0700 Subject: [PATCH 09/12] Update docker builder doc --- docs/sources/use/builder.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index 735b2e575f..84d275782e 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -107,8 +107,7 @@ The `ENV` instruction sets the environment variable `` to the value functionally equivalent to prefixing the command with `=` .. note:: - The environment variables are local to the Dockerfile, they will not persist - when a container is run from the resulting image. + The environment variables will persist when a container is run from the resulting image. 2.7 INSERT ---------- @@ -122,6 +121,8 @@ curl was installed within the image. .. note:: The path must include the file name. +.. note:: + This instruction has temporarily disabled 3. Dockerfile Examples ====================== @@ -179,4 +180,4 @@ curl was installed within the image. # Will output something like ===> 695d7793cbe4 # You'll now have two images, 907ad6c2736f with /bar, and 695d7793cbe4 with - # /oink. \ No newline at end of file + # /oink. From 3f22842542a17d41f72b4943d19d244f2296b418 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 20 May 2013 17:54:54 -0700 Subject: [PATCH 10/12] Remove no longer needed tests --- api_test.go | 35 ------------------- builder_test.go | 89 ------------------------------------------------- 2 files changed, 124 deletions(-) delete mode 100644 builder_test.go diff --git a/api_test.go b/api_test.go index 700d2c4b2c..dd685ffece 100644 --- a/api_test.go +++ b/api_test.go @@ -14,7 +14,6 @@ import ( "net/http/httptest" "os" "path" - "strings" "testing" "time" ) @@ -579,40 +578,6 @@ func TestPostCommit(t *testing.T) { } } -func TestPostBuild(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - - srv := &Server{runtime: runtime} - - imgs, err := runtime.graph.All() - if err != nil { - t.Fatal(err) - } - beginCount := len(imgs) - - req, err := http.NewRequest("POST", "/build", strings.NewReader(Dockerfile)) - if err != nil { - t.Fatal(err) - } - - r := httptest.NewRecorder() - if err := postBuild(srv, r, req, nil); err != nil { - t.Fatal(err) - } - - imgs, err = runtime.graph.All() - if err != nil { - t.Fatal(err) - } - if len(imgs) != beginCount+3 { - t.Fatalf("Expected %d images, %d found", beginCount+3, len(imgs)) - } -} - func TestPostImagesCreate(t *testing.T) { // FIXME: Use the staging in order to perform tests diff --git a/builder_test.go b/builder_test.go deleted file mode 100644 index e3a24e86ec..0000000000 --- a/builder_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package docker - -import ( - "github.com/dotcloud/docker/utils" - "strings" - "testing" -) - -const Dockerfile = ` -# VERSION 0.1 -# DOCKER-VERSION 0.2 - -from ` + unitTestImageName + ` -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md -` - -func TestBuild(t *testing.T) { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) - - builder := NewBuilder(runtime) - - img, err := builder.Build(strings.NewReader(Dockerfile), &utils.NopWriter{}) - if err != nil { - t.Fatal(err) - } - - container, err := builder.Create( - &Config{ - Image: img.Id, - Cmd: []string{"cat", "/tmp/passwd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container) - - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "root:testpass\n" { - t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") - } - - container2, err := builder.Create( - &Config{ - Image: img.Id, - Cmd: []string{"ls", "-d", "/var/run/sshd"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container2) - - output, err = container2.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "/var/run/sshd\n" { - t.Fatal("/var/run/sshd has not been created") - } - - container3, err := builder.Create( - &Config{ - Image: img.Id, - Cmd: []string{"cat", "/tmp/CHANGELOG.md"}, - }, - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container3) - - output, err = container3.Output() - if err != nil { - t.Fatal(err) - } - if len(output) == 0 { - t.Fatal("/tmp/CHANGELOG.md has not been copied") - } -} From 5818813183bfb61180846f2b1b23a6fdcb2c9cdd Mon Sep 17 00:00:00 2001 From: Christopher Currie Date: Tue, 21 May 2013 21:45:27 -0600 Subject: [PATCH 11/12] Apparent typos in the docs. --- docs/sources/examples/python_web_app.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/examples/python_web_app.rst b/docs/sources/examples/python_web_app.rst index 992a09dc42..3dd25015f1 100644 --- a/docs/sources/examples/python_web_app.rst +++ b/docs/sources/examples/python_web_app.rst @@ -40,7 +40,7 @@ We attach to the new container to see what is going on. Ctrl-C to disconnect .. code-block:: bash - BUILD_IMG=$(docker commit $BUILD_JOB _/builds/github.com/hykes/helloflask/master) + BUILD_IMG=$(docker commit $BUILD_JOB _/builds/github.com/shykes/helloflask/master) Save the changed we just made in the container to a new image called "_/builds/github.com/hykes/helloflask/master" and save the image id in the BUILD_IMG variable name. @@ -58,7 +58,7 @@ Use the new image we just created and create a new container with network port 5 .. code-block:: bash docker logs $WEB_WORKER - * Running on \http://0.0.0.0:5000/ + * Running on http://0.0.0.0:5000/ view the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output. @@ -70,7 +70,7 @@ lookup the public-facing port which is NAT-ed store the private port used by the .. code-block:: bash - curl \http://`hostname`:$WEB_PORT + curl http://`hostname`:$WEB_PORT Hello world! access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console. From 949a649cc2364f796daae14fa3b044432e25efdf Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 22 May 2013 13:49:12 +0000 Subject: [PATCH 12/12] fix content type in doc --- docs/sources/api/docker_remote_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 2b1aad0e84..5d62963b40 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -118,7 +118,8 @@ Create a container .. sourcecode:: http HTTP/1.1 201 OK - + Content-Type: application/json + { "Id":"e90e34656806" "Warnings":[]