From e2880950c5ea721be00a814d93ea256feb9f98bb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 1/9] Add build command --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index 442d7f55ee..8d8046c0f5 100644 --- a/commands.go +++ b/commands.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "os" "path/filepath" "runtime" "strconv" From ebb59c1125d194716f26b8cc53863fa9910c4b3f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 30 Apr 2013 18:03:15 -0700 Subject: [PATCH 2/9] Remove the open from CmdBuild --- commands.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands.go b/commands.go index 8d8046c0f5..d9756c9d88 100644 --- a/commands.go +++ b/commands.go @@ -10,8 +10,6 @@ import ( "log" "net/http" "net/url" - "os" - "path/filepath" "runtime" "strconv" "strings" From d92166cc7937afa39e2ed537c601555a0a5b84eb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sat, 4 May 2013 20:58:45 -0700 Subject: [PATCH 3/9] Fix merge issue --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index d9756c9d88..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "path/filepath" "runtime" "strconv" "strings" From 96069de4e0818f9d513f6ea495e03d8e67eb1e98 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 4/9] Add build command --- builder.go | 308 +++++++--------------------------------------------- commands.go | 1 + 2 files changed, 39 insertions(+), 270 deletions(-) diff --git a/builder.go b/builder.go index 72989a8a0a..4de71e017b 100644 --- a/builder.go +++ b/builder.go @@ -4,175 +4,60 @@ import ( "bufio" "fmt" "io" - "os" - "path" "strings" - "time" ) type Builder struct { - runtime *Runtime - repositories *TagStore - graph *Graph + runtime *Runtime } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, - graph: runtime.graph, - repositories: runtime.repositories, + runtime: runtime, } } -func (builder *Builder) mergeConfig(userConf, imageConf *Config) { - if userConf.Hostname != "" { - userConf.Hostname = imageConf.Hostname +func (builder *Builder) run(image *Image, cmd string) (*Container, error) { + // FIXME: pass a NopWriter instead of nil + config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + if config.Image == "" { + return nil, fmt.Errorf("Image not specified") } - if userConf.User != "" { - userConf.User = imageConf.User + if len(config.Cmd) == 0 { + return nil, fmt.Errorf("Command not specified") } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory + if config.Tty { + return nil, fmt.Errorf("The tty mode is not supported within the builder") } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { - userConf.PortSpecs = imageConf.PortSpecs - } - if !userConf.Tty { - userConf.Tty = userConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } -} -func (builder *Builder) Create(config *Config) (*Container, error) { - // Lookup image - img, err := builder.repositories.LookupImage(config.Image) + // Create new container + container, err := builder.runtime.Create(config) if err != nil { return nil, err } - - if img.Config != nil { - builder.mergeConfig(config, img.Config) - } - - if config.Cmd == nil { - return nil, fmt.Errorf("No command specified") - } - - // Generate id - id := GenerateId() - // Generate default hostname - // FIXME: the lxc template no longer needs to set a default hostname - if config.Hostname == "" { - config.Hostname = id[:12] - } - - container := &Container{ - // FIXME: we should generate the ID here instead of receiving it as an argument - Id: id, - Created: time.Now(), - Path: config.Cmd[0], - Args: config.Cmd[1:], //FIXME: de-duplicate from config - Config: config, - Image: img.Id, // Always use the resolved image id - NetworkSettings: &NetworkSettings{}, - // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, - } - container.root = builder.runtime.containerRoot(container.Id) - // Step 1: create the container directory. - // This doubles as a barrier to avoid race conditions. - if err := os.Mkdir(container.root, 0700); err != nil { - return nil, err - } - - // If custom dns exists, then create a resolv.conf for the container - if len(config.Dns) > 0 { - container.ResolvConfPath = path.Join(container.root, "resolv.conf") - f, err := os.Create(container.ResolvConfPath) - if err != nil { - return nil, err - } - defer f.Close() - for _, dns := range config.Dns { - if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { - return nil, err - } - } - } else { - container.ResolvConfPath = "/etc/resolv.conf" - } - - // Step 2: save the container json - if err := container.ToDisk(); err != nil { - return nil, err - } - // Step 3: register the container - if err := builder.runtime.Register(container); err != nil { + if err := container.Start(); err != nil { return nil, err } return container, nil } -// Commit creates a new filesystem image from the current state of a container. -// The image can optionally be tagged into a repository -func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { - // FIXME: freeze the container before copying it to avoid data corruption? - // FIXME: this shouldn't be in commands. - rwTar, err := container.ExportRw() +func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { + c, err := builder.run(image, cmd) if err != nil { return nil, err } - // Create a new image from the container's base layers + a new layer from container changes - img, err := builder.graph.Create(rwTar, container, comment, author, config) + if result := c.Wait(); result != 0 { + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + } + img, err := builder.runtime.Commit(c.Id, "", "", "", "") if err != nil { return nil, err } - // Register the image if needed - if repository != "" { - if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { - return img, err - } - } return img, nil } -func (builder *Builder) clearTmp(containers, images map[string]struct{}) { - for c := range containers { - tmp := builder.runtime.Get(c) - builder.runtime.Destroy(tmp) - Debugf("Removing container %s", c) - } - for i := range images { - builder.runtime.graph.Delete(i) - Debugf("Removing image %s", i) - } -} - -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { - var ( - image, base *Image - maintainer string - tmpContainers map[string]struct{} = make(map[string]struct{}) - tmpImages map[string]struct{} = make(map[string]struct{}) - ) - defer builder.clearTmp(tmpContainers, tmpImages) +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { + var image, base *Image file := bufio.NewReader(dockerfile) for { @@ -181,23 +66,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err == io.EOF { break } - return nil, err + return err } line = strings.TrimSpace(line) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid Dockerfile format") + return fmt.Errorf("Invalid Dockerfile format") } - instruction := strings.Trim(tmp[0], " ") - arguments := strings.Trim(tmp[1], " ") - switch strings.ToLower(instruction) { + switch tmp[0] { case "from": - fmt.Fprintf(stdout, "FROM %s\n", arguments) - image, err = builder.runtime.repositories.LookupImage(arguments) + fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) + image, err = builder.runtime.repositories.LookupImage(tmp[1]) if err != nil { if builder.runtime.graph.IsNotExist(err) { @@ -221,143 +104,28 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } } - - break - case "mainainer": - fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) - maintainer = arguments break case "run": - fmt.Fprintf(stdout, "RUN %s\n", arguments) + fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to run") + return fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) + base, err = builder.runCommit(image, tmp[1]) if err != nil { - return nil, err + return err } - - // Create the container and start it - c, err := builder.Create(config) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - // Wait for it to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - // use the base as the new image - image = base - - break - case "expose": - ports := strings.Split(arguments, " ") - - fmt.Fprintf(stdout, "EXPOSE %v\n", ports) - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - - // Create the container and start it - c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - image = base - break - case "insert": - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - tmp = strings.SplitN(arguments, " ", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid INSERT format") - } - sourceUrl := strings.Trim(tmp[0], " ") - destPath := strings.Trim(tmp[1], " ") - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - - file, err := Download(sourceUrl, stdout) - if err != nil { - return nil, err - } - defer file.Body.Close() - - config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) - if err != nil { - return nil, err - } - c, err := builder.Create(config) - if err != nil { - return nil, err - } - - if err := c.Start(); err != nil { - return nil, err - } - - // Wait for echo to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - if err := c.Inject(file.Body, destPath); err != nil { - return nil, err - } - - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - image = base - + fmt.Fprintf(stdout, "===> %s\n", base.Id) break + case "copy": + return fmt.Errorf("The copy operator has not yet been implemented") default: - fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", instruction) + fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } } if base != nil { - // The build is successful, keep the temporary containers and images - for i := range tmpImages { - delete(tmpImages, i) - } - for i := range tmpContainers { - delete(tmpContainers, i) - } - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.ShortId()) + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) } else { fmt.Fprintf(stdout, "An error occured during the build\n") } - return base, nil + return nil } diff --git a/commands.go b/commands.go index 442d7f55ee..c72e24aa61 100644 --- a/commands.go +++ b/commands.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path/filepath" + "os" "runtime" "strconv" "strings" From f911ccc27bcbcf281a0d690c5d7b05709c8947d4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 5/9] Moving runtime.Create to builder.Create --- builder.go | 149 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 21 deletions(-) diff --git a/builder.go b/builder.go index 4de71e017b..789f8642eb 100644 --- a/builder.go +++ b/builder.go @@ -4,38 +4,80 @@ import ( "bufio" "fmt" "io" + "os" + "path" "strings" + "time" ) type Builder struct { - runtime *Runtime + runtime *Runtime + repositories *TagStore } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, + runtime: runtime, + repositories: runtime.repositories, } } -func (builder *Builder) run(image *Image, cmd string) (*Container, error) { - // FIXME: pass a NopWriter instead of nil - config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) - if config.Image == "" { - return nil, fmt.Errorf("Image not specified") - } - if len(config.Cmd) == 0 { - return nil, fmt.Errorf("Command not specified") - } - if config.Tty { - return nil, fmt.Errorf("The tty mode is not supported within the builder") - } - - // Create new container - container, err := builder.runtime.Create(config) +func (builder *Builder) Create(config *Config) (*Container, error) { + // Lookup image + img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } - if err := container.Start(); err != nil { + // Generate id + id := GenerateId() + // Generate default hostname + // FIXME: the lxc template no longer needs to set a default hostname + if config.Hostname == "" { + config.Hostname = id[:12] + } + + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: id, + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = builder.runtime.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + + // If custom dns exists, then create a resolv.conf for the container + if len(config.Dns) > 0 { + container.ResolvConfPath = path.Join(container.root, "resolv.conf") + f, err := os.Create(container.ResolvConfPath) + if err != nil { + return nil, err + } + defer f.Close() + for _, dns := range config.Dns { + if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { + return nil, err + } + } + } else { + container.ResolvConfPath = "/etc/resolv.conf" + } + + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := builder.runtime.Register(container); err != nil { return nil, err } return container, nil @@ -110,14 +152,79 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } - base, err = builder.runCommit(image, tmp[1]) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) if err != nil { return err } - fmt.Fprintf(stdout, "===> %s\n", base.Id) + + // Create the container and start it + c, err := builder.Create(config) + if err != nil { + return err + } + if err := c.Start(); err != nil { + return err + } + tmpContainers[c.Id] = struct{}{} + + // Wait for it to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + // Commit the container + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break case "copy": - return fmt.Errorf("The copy operator has not yet been implemented") + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp2 := strings.SplitN(tmp[1], " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid COPY format") + } + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + + file, err := Download(tmp2[0], stdout) + if err != nil { + return err + } + defer file.Body.Close() + + config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } + c, err := builder.Create(config) + if err != nil { + return err + } + + if err := c.Start(); err != nil { + return err + } + + // Wait for echo to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + if err := c.Inject(file.Body, tmp2[1]); err != nil { + return err + } + + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } From 9959e2cd637d09e8c37891cebcb1764c825ef7d5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:36:45 -0700 Subject: [PATCH 6/9] Rebase master (autorun) --- builder.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/builder.go b/builder.go index 789f8642eb..5ac4fa3c36 100644 --- a/builder.go +++ b/builder.go @@ -22,12 +22,57 @@ func NewBuilder(runtime *Runtime) *Builder { } } +func (builder *Builder) mergeConfig(userConf, imageConf *Config) { + if userConf.Hostname != "" { + userConf.Hostname = imageConf.Hostname + } + if userConf.User != "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { + userConf.PortSpecs = imageConf.PortSpecs + } + if !userConf.Tty { + userConf.Tty = userConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } +} + func (builder *Builder) Create(config *Config) (*Container, error) { // Lookup image img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } + + if img.Config != nil { + builder.mergeConfig(config, img.Config) + } + + if config.Cmd == nil { + return nil, fmt.Errorf("No command specified") + } + // Generate id id := GenerateId() // Generate default hostname @@ -83,15 +128,17 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return container, nil } -func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { - c, err := builder.run(image, cmd) +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() if err != nil { return nil, err } - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) - } - img, err := builder.runtime.Commit(c.Id, "", "", "", "") + // Create a new image from the container's base layers + a new layer from container changes + img, err := builder.graph.Create(rwTar, container, comment, author, config) if err != nil { return nil, err } @@ -173,7 +220,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return err } @@ -219,7 +266,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { return err } - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return err } From a46fc3a59e585e005ec67b05211ca9de02f0c4ff Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 00:49:23 -0700 Subject: [PATCH 7/9] Implement caching for docker builder --- builder.go | 75 +++++++++++++++++++++++++++++++++++++++++++++++++----- utils.go | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/builder.go b/builder.go index 5ac4fa3c36..e7f5e330f7 100644 --- a/builder.go +++ b/builder.go @@ -145,8 +145,55 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a return img, nil } -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { - var image, base *Image +func (builder *Builder) clearTmp(containers, images map[string]struct{}) { + for c := range containers { + tmp := builder.runtime.Get(c) + builder.runtime.Destroy(tmp) + Debugf("Removing container %s", c) + } + for i := range images { + builder.runtime.graph.Delete(i) + Debugf("Removing image %s", i) + } +} + +func (builder *Builder) getCachedImage(image *Image, config *Config) (*Image, error) { + // Retrieve all images + images, err := builder.graph.All() + if err != nil { + return nil, err + } + + // Store the tree in a map of map (map[parentId][childId]) + imageMap := make(map[string]map[string]struct{}) + for _, img := range images { + if _, exists := imageMap[img.Parent]; !exists { + imageMap[img.Parent] = make(map[string]struct{}) + } + imageMap[img.Parent][img.Id] = struct{}{} + } + + // Loop on the children of the given image and check the config + for elem := range imageMap[image.Id] { + img, err := builder.graph.Get(elem) + if err != nil { + return nil, err + } + if CompareConfig(&img.ContainerConfig, config) { + return img, nil + } + } + return nil, nil +} + +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { + var ( + image, base *Image + maintainer string + tmpContainers map[string]struct{} = make(map[string]struct{}) + tmpImages map[string]struct{} = make(map[string]struct{}) + ) + defer builder.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -204,6 +251,14 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { return err } + if cache, err := builder.getCachedImage(image, config); err != nil { + return nil, err + } else if cache != nil { + image = cache + fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) + break + } + // Create the container and start it c, err := builder.Create(config) if err != nil { @@ -276,10 +331,16 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } } - if base != nil { - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) - } else { - fmt.Fprintf(stdout, "An error occured during the build\n") + if image != nil { + // The build is successful, keep the temporary containers and images + for i := range tmpImages { + delete(tmpImages, i) + } + for i := range tmpContainers { + delete(tmpContainers, i) + } + fmt.Fprintf(stdout, "Build finished. image id: %s\n", image.ShortId()) + return image, nil } - return nil + return nil, fmt.Errorf("An error occured during the build\n") } diff --git a/utils.go b/utils.go index 047a29abef..095be2f4bf 100644 --- a/utils.go +++ b/utils.go @@ -474,3 +474,50 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } + +// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields +// If OpenStdin is set, then it differs +func CompareConfig(a, b *Config) bool { + if a == nil || b == nil || + a.OpenStdin || b.OpenStdin { + return false + } + if a.AttachStdout != b.AttachStdout || + a.AttachStderr != b.AttachStderr || + a.User != b.User || + a.Memory != b.Memory || + a.MemorySwap != b.MemorySwap || + a.OpenStdin != b.OpenStdin || + a.Tty != b.Tty { + return false + } + if len(a.Cmd) != len(b.Cmd) || + len(a.Dns) != len(b.Dns) || + len(a.Env) != len(b.Env) || + len(a.PortSpecs) != len(b.PortSpecs) { + return false + } + + for i := 0; i < len(a.Cmd); i++ { + if a.Cmd[i] != b.Cmd[i] { + return false + } + } + for i := 0; i < len(a.Dns); i++ { + if a.Dns[i] != b.Dns[i] { + return false + } + } + for i := 0; i < len(a.Env); i++ { + if a.Env[i] != b.Env[i] { + return false + } + } + for i := 0; i < len(a.PortSpecs); i++ { + if a.PortSpecs[i] != b.PortSpecs[i] { + return false + } + } + + return true +} From 756df27e45f2e5d9033058f60afa9547239af1d6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 01:18:48 -0700 Subject: [PATCH 8/9] Add compatibility with contrib builder --- builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder.go b/builder.go index e7f5e330f7..19c24c901e 100644 --- a/builder.go +++ b/builder.go @@ -204,7 +204,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } return err } - line = strings.TrimSpace(line) + line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue @@ -328,7 +328,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break default: - fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) } } if image != nil { From 35c59f4e05fb8dc3a5939d6af8c9313ebda6a5dd Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 16:58:09 -0700 Subject: [PATCH 9/9] Rebase fix --- builder.go | 132 ++++++++++++++++++++++++++++++++++++---------------- commands.go | 1 - 2 files changed, 93 insertions(+), 40 deletions(-) diff --git a/builder.go b/builder.go index 19c24c901e..84fb01eb49 100644 --- a/builder.go +++ b/builder.go @@ -13,11 +13,13 @@ import ( type Builder struct { runtime *Runtime repositories *TagStore + graph *Graph } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ runtime: runtime, + graph: runtime.graph, repositories: runtime.repositories, } } @@ -142,6 +144,12 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a if err != nil { return nil, err } + // Register the image if needed + if repository != "" { + if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { + return img, err + } + } return img, nil } @@ -202,29 +210,33 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err == io.EOF { break } - return err + return nil, err } line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid Dockerfile format") + return nil, fmt.Errorf("Invalid Dockerfile format") } - switch tmp[0] { + instruction := strings.Trim(tmp[0], " ") + arguments := strings.Trim(tmp[1], " ") + switch strings.ToLower(instruction) { case "from": - fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) - image, err = builder.runtime.repositories.LookupImage(tmp[1]) + fmt.Fprintf(stdout, "FROM %s\n", arguments) + image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { if builder.runtime.graph.IsNotExist(err) { var tag, remote string - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") + if strings.Contains(arguments, ":") { + remoteParts := strings.Split(arguments, ":") tag = remoteParts[1] remote = remoteParts[0] + } else { + remote = arguments } if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { @@ -235,20 +247,24 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err != nil { return nil, err } - } else { return nil, err } } + + break + case "mainainer": + fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) + maintainer = arguments break case "run": - fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + fmt.Fprintf(stdout, "RUN %s\n", arguments) if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to run") + return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } if cache, err := builder.getCachedImage(image, config); err != nil { @@ -262,70 +278,108 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Create the container and start it c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } tmpContainers[c.Id] = struct{}{} // Wait for it to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } // Commit the container - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { - return err + return nil, err } tmpImages[base.Id] = struct{}{} fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - break - case "copy": - if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to copy") - } - tmp2 := strings.SplitN(tmp[1], " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid COPY format") - } - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) - file, err := Download(tmp2[0], stdout) + // use the base as the new image + image = base + + break + case "expose": + ports := strings.Split(arguments, " ") + + fmt.Fprintf(stdout, "EXPOSE %v\n", ports) + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + + // Create the container and start it + c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) if err != nil { - return err + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + tmpContainers[c.Id] = struct{}{} + + // Commit the container + base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) + if err != nil { + return nil, err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break + case "insert": + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp = strings.SplitN(arguments, " ", 2) + if len(tmp) != 2 { + return nil, fmt.Errorf("Invalid INSERT format") + } + sourceUrl := strings.Trim(tmp[0], " ") + destPath := strings.Trim(tmp[1], " ") + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) + + file, err := Download(sourceUrl, stdout) + if err != nil { + return nil, err } defer file.Body.Close() - config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } // Wait for echo to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } - if err := c.Inject(file.Body, tmp2[1]); err != nil { - return err + if err := c.Inject(file.Body, destPath); err != nil { + return nil, err } - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { - return err + return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break default: fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) diff --git a/commands.go b/commands.go index c72e24aa61..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "path/filepath" - "os" "runtime" "strconv" "strings"