rebase master

AUTHORS

@@ -29,6 +29,7 @@ Dr Nic Williams <drnicwilliams@gmail.com>
 Elias Probst <mail@eliasprobst.eu>
 Eric Hanchrow <ehanchrow@ine.com>
 Evan Wies <evan@neomantra.net>
+Eric Myhre <hash@exultant.us>
 ezbercih <cem.ezberci@gmail.com>
 Flavio Castelli <fcastelli@suse.com>
 Francisco Souza <f@souza.cc>

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 0.4.7 (2013-06-28)
+ * Registry: easier push/pull to a custom registry
+ * Remote API: the progress bar updates faster when downloading and uploading large files
+ - Remote API: fix a bug in the optional unix socket transport
+ * Runtime: improve detection of kernel version
+ + Runtime: host directories can be mounted as volumes with 'docker run -b'
+ - Runtime: fix an issue when only attaching to stdin
+ * Runtime: use 'tar --numeric-owner' to avoid uid mismatch across multiple hosts
+ * Hack: improve test suite and dev environment
+ * Hack: remove dependency on unit tests on 'os/user'
+ + Documentation: add terminology section
+
+## 0.4.6 (2013-06-22)
+ - Runtime: fix a bug which caused creation of empty images (and volumes) to crash.
+
+## 0.4.5 (2013-06-21)
+ + Builder: 'docker build git://URL' fetches and builds a remote git repository
+ * Runtime: 'docker ps -s' optionally prints container size
+ * Tests: Improved and simplified
+ - Runtime: fix a regression introduced in 0.4.3 which caused the logs command to fail.
+ - Builder: fix a regression when using ADD with single regular file.
+
 ## 0.4.4 (2013-06-19)
  - Builder: fix a regression introduced in 0.4.3 which caused builds to fail on new clients.
 

FIXME

@@ -33,3 +33,4 @@ to put them - so we put them here :)
 * Caching after an ADD
 * entry point config
 * bring back git revision info, looks like it was lost
+* Clean up the ProgressReader api, it's a PITA to use

Makefile

@@ -2,6 +2,8 @@ DOCKER_PACKAGE := github.com/dotcloud/docker
 RELEASE_VERSION := $(shell git tag | grep -E "v[0-9\.]+$$" | sort -nr | head -n 1)
 SRCRELEASE := docker-$(RELEASE_VERSION)
 BINRELEASE := docker-$(RELEASE_VERSION).tgz
+BUILD_SRC := build_src
+BUILD_PATH := ${BUILD_SRC}/src/${DOCKER_PACKAGE}
 
 GIT_ROOT := $(shell git rev-parse --show-toplevel)
 BUILD_DIR := $(CURDIR)/.gopath
@@ -71,8 +73,13 @@ else ifneq ($(DOCKER_DIR), $(realpath $(DOCKER_DIR)))
 	@rm -f $(DOCKER_DIR)
 endif
 
-test: all
+test:
-	@(cd $(DOCKER_DIR); sudo -E go test $(GO_OPTIONS))
+	# Copy docker source and dependencies for testing
+	rm -rf ${BUILD_SRC}; mkdir -p ${BUILD_PATH}
+	tar --exclude=${BUILD_SRC} -cz . | tar -xz -C ${BUILD_PATH}
+	GOPATH=${CURDIR}/${BUILD_SRC} go get -d
+	# Do the test
+	sudo -E GOPATH=${CURDIR}/${BUILD_SRC} go test ${GO_OPTIONS}
 
 testall: all
 	@(cd $(DOCKER_DIR); sudo -E go test ./... $(GO_OPTIONS))

api.go

@@ -7,15 +7,17 @@ import (
 	"github.com/dotcloud/docker/utils"
 	"github.com/gorilla/mux"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 )
 
-const APIVERSION = 1.2
+const APIVERSION = 1.3
 const DEFAULTHTTPHOST string = "127.0.0.1"
 const DEFAULTHTTPPORT int = 4243
 
@@ -67,16 +69,16 @@ func writeJSON(w http.ResponseWriter, b []byte) {
 	w.Write(b)
 }
 
-// FIXME: Use stvconv.ParseBool() instead?
 func getBoolParam(value string) (bool, error) {
-	if value == "1" || strings.ToLower(value) == "true" {
+	if value == "" {
-		return true, nil
-	}
-	if value == "" || value == "0" || strings.ToLower(value) == "false" {
 		return false, nil
 	}
+	ret, err := strconv.ParseBool(value)
+	if err != nil {
 		return false, fmt.Errorf("Bad parameter")
 	}
+	return ret, nil
+}
 
 func getAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if version > 1.1 {
@@ -256,6 +258,10 @@ func getContainersJSON(srv *Server, version float64, w http.ResponseWriter, r *h
 	if err != nil {
 		return err
 	}
+	size, err := getBoolParam(r.Form.Get("size"))
+	if err != nil {
+		return err
+	}
 	since := r.Form.Get("since")
 	before := r.Form.Get("before")
 	n, err := strconv.Atoi(r.Form.Get("limit"))
@@ -263,7 +269,7 @@ func getContainersJSON(srv *Server, version float64, w http.ResponseWriter, r *h
 		n = -1
 	}
 
-	outs := srv.Containers(all, n, since, before)
+	outs := srv.Containers(all, size, n, since, before)
 	b, err := json.Marshal(outs)
 	if err != nil {
 		return err
@@ -529,7 +535,7 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R
 		return err
 	}
 	if imgs != nil {
-		if len(*imgs) != 0 {
+		if len(imgs) != 0 {
 			b, err := json.Marshal(imgs)
 			if err != nil {
 				return err
@@ -545,11 +551,20 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R
 }
 
 func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	hostConfig := &HostConfig{}
+
+	// allow a nil body for backwards compatibility
+	if r.Body != nil {
+		if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil {
+			return err
+		}
+	}
+
 	if vars == nil {
 		return fmt.Errorf("Missing parameter")
 	}
 	name := vars["name"]
-	if err := srv.ContainerStart(name); err != nil {
+	if err := srv.ContainerStart(name, hostConfig); err != nil {
 		return err
 	}
 	w.WriteHeader(http.StatusNoContent)
@@ -654,7 +669,20 @@ func postContainersAttach(srv *Server, version float64, w http.ResponseWriter, r
 	if err != nil {
 		return err
 	}
-	defer in.Close()
+	defer func() {
+		if tcpc, ok := in.(*net.TCPConn); ok {
+			tcpc.CloseWrite()
+		} else {
+			in.Close()
+		}
+	}()
+	defer func() {
+		if tcpc, ok := out.(*net.TCPConn); ok {
+			tcpc.CloseWrite()
+		} else if closer, ok := out.(io.Closer); ok {
+			closer.Close()
+		}
+	}()
 
 	fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n")
 	if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, in, out); err != nil {
@@ -723,34 +751,65 @@ func postImagesGetCache(srv *Server, version float64, w http.ResponseWriter, r *
 }
 
 func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	if err := r.ParseMultipartForm(4096); err != nil {
+	if version < 1.3 {
-		return err
+		return fmt.Errorf("Multipart upload for build is no longer supported. Please upgrade your docker client.")
 	}
-	remote := r.FormValue("t")
+	remoteURL := r.FormValue("remote")
+	repoName := r.FormValue("t")
 	tag := ""
-	if strings.Contains(remote, ":") {
+	if strings.Contains(repoName, ":") {
-		remoteParts := strings.Split(remote, ":")
+		remoteParts := strings.Split(repoName, ":")
 		tag = remoteParts[1]
-		remote = remoteParts[0]
+		repoName = remoteParts[0]
 	}
 
-	dockerfile, _, err := r.FormFile("Dockerfile")
+	var context io.Reader
+
+	if remoteURL == "" {
+		context = r.Body
+	} else if utils.IsGIT(remoteURL) {
+		if !strings.HasPrefix(remoteURL, "git://") {
+			remoteURL = "https://" + remoteURL
+		}
+		root, err := ioutil.TempDir("", "docker-build-git")
 		if err != nil {
 			return err
 		}
+		defer os.RemoveAll(root)
 
-	context, _, err := r.FormFile("Context")
+		if output, err := exec.Command("git", "clone", remoteURL, root).CombinedOutput(); err != nil {
+			return fmt.Errorf("Error trying to use git: %s (%s)", err, output)
+		}
+
+		c, err := Tar(root, Bzip2)
 		if err != nil {
-		if err != http.ErrMissingFile {
 			return err
 		}
+		context = c
+	} else if utils.IsURL(remoteURL) {
+		f, err := utils.Download(remoteURL, ioutil.Discard)
+		if err != nil {
+			return err
+		}
+		defer f.Body.Close()
+		dockerFile, err := ioutil.ReadAll(f.Body)
+		if err != nil {
+			return err
+		}
+		c, err := mkBuildContext(string(dockerFile), nil)
+		if err != nil {
+			return err
+		}
+		context = c
 	}
-
 	b := NewBuildFile(srv, utils.NewWriteFlusher(w))
-	if id, err := b.Build(dockerfile, context); err != nil {
+	id, err := b.Build(context)
+	if err != nil {
 		fmt.Fprintf(w, "Error build: %s\n", err)
-	} else if remote != "" {
+		return err
-		srv.runtime.repositories.Set(remote, tag, id, false)
+	}
+	if repoName != "" {
+		srv.runtime.repositories.Set(repoName, tag, id, false)
 	}
 	return nil
 }

api_test.go

@@ -17,6 +17,30 @@ import (
 	"time"
 )
 
+func TestGetBoolParam(t *testing.T) {
+	if ret, err := getBoolParam("true"); err != nil || !ret {
+		t.Fatalf("true -> true, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam("True"); err != nil || !ret {
+		t.Fatalf("True -> true, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam("1"); err != nil || !ret {
+		t.Fatalf("1 -> true, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam(""); err != nil || ret {
+		t.Fatalf("\"\" -> false, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam("false"); err != nil || ret {
+		t.Fatalf("false -> false, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam("0"); err != nil || ret {
+		t.Fatalf("0 -> false, nil | got %t %s", ret, err)
+	}
+	if ret, err := getBoolParam("faux"); err == nil || ret {
+		t.Fatalf("faux -> false, err | got %t %s", ret, err)
+	}
+}
+
 func TestPostAuth(t *testing.T) {
 	runtime, err := newTestRuntime()
 	if err != nil {
@@ -849,7 +873,8 @@ func TestPostContainersKill(t *testing.T) {
 	}
 	defer runtime.Destroy(container)
 
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -893,7 +918,8 @@ func TestPostContainersRestart(t *testing.T) {
 	}
 	defer runtime.Destroy(container)
 
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -949,8 +975,15 @@ func TestPostContainersStart(t *testing.T) {
 	}
 	defer runtime.Destroy(container)
 
+	hostConfigJSON, err := json.Marshal(&HostConfig{})
+
+	req, err := http.NewRequest("POST", "/containers/"+container.ID+"/start", bytes.NewReader(hostConfigJSON))
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	r := httptest.NewRecorder()
-	if err := postContainersStart(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil {
+	if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil {
 		t.Fatal(err)
 	}
 	if r.Code != http.StatusNoContent {
@@ -965,7 +998,7 @@ func TestPostContainersStart(t *testing.T) {
 	}
 
 	r = httptest.NewRecorder()
-	if err = postContainersStart(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err == nil {
+	if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err == nil {
 		t.Fatalf("A running containter should be able to be started")
 	}
 
@@ -995,7 +1028,8 @@ func TestPostContainersStop(t *testing.T) {
 	}
 	defer runtime.Destroy(container)
 
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1044,7 +1078,8 @@ func TestPostContainersWait(t *testing.T) {
 	}
 	defer runtime.Destroy(container)
 
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1089,7 +1124,8 @@ func TestPostContainersAttach(t *testing.T) {
 	defer runtime.Destroy(container)
 
 	// Start the process
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 

archive.go

@@ -1,7 +1,9 @@
 package docker
 
 import (
+	"archive/tar"
 	"bufio"
+	"bytes"
 	"errors"
 	"fmt"
 	"github.com/dotcloud/docker/utils"
@@ -10,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 )
 
 type Archive io.Reader
@@ -89,7 +92,7 @@ func Tar(path string, compression Compression) (io.Reader, error) {
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
 func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
-	args := []string{"tar", "-f", "-", "-C", path}
+	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
 	if filter == nil {
 		filter = []string{"."}
 	}
@@ -105,7 +108,9 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
 func Untar(archive io.Reader, path string) error {
-
+	if archive == nil {
+		return fmt.Errorf("Empty archive")
+	}
 	bufferedArchive := bufio.NewReaderSize(archive, 10)
 	buf, err := bufferedArchive.Peek(10)
 	if err != nil {
@@ -115,7 +120,7 @@ func Untar(archive io.Reader, path string) error {
 
 	utils.Debugf("Archive compression detected: %s", compression.Extension())
 
-	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
+	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
 	cmd.Stdin = bufferedArchive
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
@@ -160,51 +165,60 @@ func CopyWithTar(src, dst string) error {
 	if err != nil {
 		return err
 	}
-	var dstExists bool
-	dstSt, err := os.Stat(dst)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-	} else {
-		dstExists = true
-	}
-	// Things that can go wrong if the source is a directory
-	if srcSt.IsDir() {
-		// The destination exists and is a regular file
-		if dstExists && !dstSt.IsDir() {
-			return fmt.Errorf("Can't copy a directory over a regular file")
-		}
-		// Things that can go wrong if the source is a regular file
-	} else {
-		utils.Debugf("The destination exists, it's a directory, and doesn't end in /")
-		// The destination exists, it's a directory, and doesn't end in /
-		if dstExists && dstSt.IsDir() && dst[len(dst)-1] != '/' {
-			return fmt.Errorf("Can't copy a regular file over a directory %s |%s|", dst, dst[len(dst)-1])
-		}
-	}
-	// Create the destination
-	var dstDir string
-	if srcSt.IsDir() || dst[len(dst)-1] == '/' {
-		// The destination ends in /, or the source is a directory
-		//   --> dst is the holding directory and needs to be created for -C
-		dstDir = dst
-	} else {
-		// The destination doesn't end in /
-		//   --> dst is the file
-		dstDir = path.Dir(dst)
-	}
-	if !dstExists {
-		// Create the holding directory if necessary
-		utils.Debugf("Creating the holding directory %s", dstDir)
-		if err := os.MkdirAll(dstDir, 0700); err != nil && !os.IsExist(err) {
-			return err
-		}
-	}
 	if !srcSt.IsDir() {
-		return TarUntar(path.Dir(src), []string{path.Base(src)}, dstDir)
+		return CopyFileWithTar(src, dst)
 	}
-	return TarUntar(src, nil, dstDir)
+	// Create dst, copy src's content into it
+	utils.Debugf("Creating dest directory: %s", dst)
+	if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
+	return TarUntar(src, nil, dst)
+}
+
+// CopyFileWithTar emulates the behavior of the 'cp' command-line
+// for a single file. It copies a regular file from path `src` to
+// path `dst`, and preserves all its metadata.
+//
+// If `dst` ends with a trailing slash '/', the final destination path
+// will be `dst/base(src)`.
+func CopyFileWithTar(src, dst string) error {
+	utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
+	srcSt, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if srcSt.IsDir() {
+		return fmt.Errorf("Can't copy a directory")
+	}
+	// Clean up the trailing /
+	if dst[len(dst)-1] == '/' {
+		dst = path.Join(dst, filepath.Base(src))
+	}
+	// Create the holding directory if necessary
+	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	hdr, err := tar.FileInfoHeader(srcSt, "")
+	if err != nil {
+		return err
+	}
+	hdr.Name = filepath.Base(dst)
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	srcF, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(tw, srcF); err != nil {
+		return err
+	}
+	tw.Close()
+	return Untar(buf, filepath.Dir(dst))
 }
 
 // CmdStream executes a command, and returns its stdout as a stream.

auth/auth.go

@@ -74,7 +74,6 @@ func decodeAuth(authStr string) (*AuthConfig, error) {
 	}
 	password := strings.Trim(arr[1], "\x00")
 	return &AuthConfig{Username: arr[0], Password: password}, nil
-
 }
 
 // load up the auth config information and return values

builder_client.go

@@ -1,314 +0,0 @@
-package docker
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"github.com/dotcloud/docker/utils"
-	"io"
-	"net/url"
-	"os"
-	"reflect"
-	"strings"
-)
-
-type builderClient struct {
-	cli *DockerCli
-
-	image      string
-	maintainer string
-	config     *Config
-
-	tmpContainers map[string]struct{}
-	tmpImages     map[string]struct{}
-
-	needCommit bool
-}
-
-func (b *builderClient) clearTmp(containers, images map[string]struct{}) {
-	for i := range images {
-		if _, _, err := b.cli.call("DELETE", "/images/"+i, nil); err != nil {
-			utils.Debugf("%s", err)
-		}
-		utils.Debugf("Removing image %s", i)
-	}
-}
-
-func (b *builderClient) CmdFrom(name string) error {
-	obj, statusCode, err := b.cli.call("GET", "/images/"+name+"/json", nil)
-	if statusCode == 404 {
-
-		remote := name
-		var tag string
-		if strings.Contains(remote, ":") {
-			remoteParts := strings.Split(remote, ":")
-			tag = remoteParts[1]
-			remote = remoteParts[0]
-		}
-		var out io.Writer
-		if os.Getenv("DEBUG") != "" {
-			out = os.Stdout
-		} else {
-			out = &utils.NopWriter{}
-		}
-		if err := b.cli.stream("POST", "/images/create?fromImage="+remote+"&tag="+tag, nil, out); err != nil {
-			return err
-		}
-		obj, _, err = b.cli.call("GET", "/images/"+name+"/json", nil)
-		if err != nil {
-			return err
-		}
-	}
-	if err != nil {
-		return err
-	}
-
-	img := &APIID{}
-	if err := json.Unmarshal(obj, img); err != nil {
-		return err
-	}
-	b.image = img.ID
-	utils.Debugf("Using image %s", b.image)
-	return nil
-}
-
-func (b *builderClient) CmdMaintainer(name string) error {
-	b.needCommit = true
-	b.maintainer = name
-	return nil
-}
-
-func (b *builderClient) CmdRun(args string) error {
-	if b.image == "" {
-		return fmt.Errorf("Please provide a source image with `from` prior to run")
-	}
-	config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil)
-	if err != nil {
-		return err
-	}
-
-	cmd, env := b.config.Cmd, b.config.Env
-	b.config.Cmd = nil
-	MergeConfig(b.config, config)
-
-	body, statusCode, err := b.cli.call("POST", "/images/getCache", &APIImageConfig{ID: b.image, Config: b.config})
-	if err != nil {
-		if statusCode != 404 {
-			return err
-		}
-	}
-	if statusCode != 404 {
-		apiID := &APIID{}
-		if err := json.Unmarshal(body, apiID); err != nil {
-			return err
-		}
-		utils.Debugf("Use cached version")
-		b.image = apiID.ID
-		return nil
-	}
-	cid, err := b.run()
-	if err != nil {
-		return err
-	}
-	b.config.Cmd, b.config.Env = cmd, env
-	return b.commit(cid)
-}
-
-func (b *builderClient) CmdEnv(args string) error {
-	b.needCommit = true
-	tmp := strings.SplitN(args, " ", 2)
-	if len(tmp) != 2 {
-		return fmt.Errorf("Invalid ENV format")
-	}
-	key := strings.Trim(tmp[0], " ")
-	value := strings.Trim(tmp[1], " ")
-
-	for i, elem := range b.config.Env {
-		if strings.HasPrefix(elem, key+"=") {
-			b.config.Env[i] = key + "=" + value
-			return nil
-		}
-	}
-	b.config.Env = append(b.config.Env, key+"="+value)
-	return nil
-}
-
-func (b *builderClient) CmdCmd(args string) error {
-	b.needCommit = true
-	var cmd []string
-	if err := json.Unmarshal([]byte(args), &cmd); err != nil {
-		utils.Debugf("Error unmarshalling: %s, using /bin/sh -c", err)
-		b.config.Cmd = []string{"/bin/sh", "-c", args}
-	} else {
-		b.config.Cmd = cmd
-	}
-	return nil
-}
-
-func (b *builderClient) CmdExpose(args string) error {
-	ports := strings.Split(args, " ")
-	b.config.PortSpecs = append(ports, b.config.PortSpecs...)
-	return nil
-}
-
-func (b *builderClient) CmdInsert(args string) error {
-	// tmp := strings.SplitN(args, "\t ", 2)
-	// sourceUrl, destPath := tmp[0], tmp[1]
-
-	// v := url.Values{}
-	// v.Set("url", sourceUrl)
-	// v.Set("path", destPath)
-	// body, _, err := b.cli.call("POST", "/images/insert?"+v.Encode(), nil)
-	// if err != nil {
-	// 	return err
-	// }
-
-	// apiId := &APIId{}
-	// if err := json.Unmarshal(body, apiId); err != nil {
-	// 	return err
-	// }
-
-	// FIXME: Reimplement this, we need to retrieve the resulting Id
-	return fmt.Errorf("INSERT not implemented")
-}
-
-func (b *builderClient) run() (string, error) {
-	if b.image == "" {
-		return "", fmt.Errorf("Please provide a source image with `from` prior to run")
-	}
-	b.config.Image = b.image
-	body, _, err := b.cli.call("POST", "/containers/create", b.config)
-	if err != nil {
-		return "", err
-	}
-
-	apiRun := &APIRun{}
-	if err := json.Unmarshal(body, apiRun); err != nil {
-		return "", err
-	}
-	for _, warning := range apiRun.Warnings {
-		fmt.Fprintln(os.Stderr, "WARNING: ", warning)
-	}
-
-	//start the container
-	_, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/start", nil)
-	if err != nil {
-		return "", err
-	}
-	b.tmpContainers[apiRun.ID] = struct{}{}
-
-	// Wait for it to finish
-	body, _, err = b.cli.call("POST", "/containers/"+apiRun.ID+"/wait", nil)
-	if err != nil {
-		return "", err
-	}
-	apiWait := &APIWait{}
-	if err := json.Unmarshal(body, apiWait); err != nil {
-		return "", err
-	}
-	if apiWait.StatusCode != 0 {
-		return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, apiWait.StatusCode)
-	}
-
-	return apiRun.ID, nil
-}
-
-func (b *builderClient) commit(id string) error {
-	if b.image == "" {
-		return fmt.Errorf("Please provide a source image with `from` prior to run")
-	}
-	b.config.Image = b.image
-
-	if id == "" {
-		cmd := b.config.Cmd
-		b.config.Cmd = []string{"true"}
-		cid, err := b.run()
-		if err != nil {
-			return err
-		}
-		id = cid
-		b.config.Cmd = cmd
-	}
-
-	// Commit the container
-	v := url.Values{}
-	v.Set("container", id)
-	v.Set("author", b.maintainer)
-
-	body, _, err := b.cli.call("POST", "/commit?"+v.Encode(), b.config)
-	if err != nil {
-		return err
-	}
-	apiID := &APIID{}
-	if err := json.Unmarshal(body, apiID); err != nil {
-		return err
-	}
-	b.tmpImages[apiID.ID] = struct{}{}
-	b.image = apiID.ID
-	b.needCommit = false
-	return nil
-}
-
-func (b *builderClient) Build(dockerfile, context io.Reader) (string, error) {
-	defer b.clearTmp(b.tmpContainers, b.tmpImages)
-	file := bufio.NewReader(dockerfile)
-	for {
-		line, err := file.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			return "", err
-		}
-		line = strings.Replace(strings.TrimSpace(line), "	", " ", 1)
-		// Skip comments and empty line
-		if len(line) == 0 || line[0] == '#' {
-			continue
-		}
-		tmp := strings.SplitN(line, " ", 2)
-		if len(tmp) != 2 {
-			return "", fmt.Errorf("Invalid Dockerfile format")
-		}
-		instruction := strings.ToLower(strings.Trim(tmp[0], " "))
-		arguments := strings.Trim(tmp[1], " ")
-
-		fmt.Fprintf(os.Stderr, "%s %s (%s)\n", strings.ToUpper(instruction), arguments, b.image)
-
-		method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:]))
-		if !exists {
-			fmt.Fprintf(os.Stderr, "Skipping unknown instruction %s\n", strings.ToUpper(instruction))
-		}
-		ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface()
-		if ret != nil {
-			return "", ret.(error)
-		}
-
-		fmt.Fprintf(os.Stderr, "===> %v\n", b.image)
-	}
-	if b.needCommit {
-		if err := b.commit(""); err != nil {
-			return "", err
-		}
-	}
-	if b.image != "" {
-		// The build is successful, keep the temporary containers and images
-		for i := range b.tmpImages {
-			delete(b.tmpImages, i)
-		}
-		for i := range b.tmpContainers {
-			delete(b.tmpContainers, i)
-		}
-		fmt.Fprintf(os.Stderr, "Build finished. image id: %s\n", b.image)
-		return b.image, nil
-	}
-	return "", fmt.Errorf("An error occured during the build\n")
-}
-
-func NewBuilderClient(proto, addr string) BuildFile {
-	return &builderClient{
-		cli:           NewDockerCli(proto, addr),
-		config:        &Config{},
-		tmpContainers: make(map[string]struct{}),
-		tmpImages:     make(map[string]struct{}),
-	}
-}

buildfile.go

@@ -14,7 +14,7 @@ import (
 )
 
 type BuildFile interface {
-	Build(io.Reader, io.Reader) (string, error)
+	Build(io.Reader) (string, error)
 	CmdFrom(string) error
 	CmdRun(string) error
 }
@@ -87,7 +87,7 @@ func (b *buildFile) CmdRun(args string) error {
 	if b.image == "" {
 		return fmt.Errorf("Please provide a source image with `from` prior to run")
 	}
-	config, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil)
+	config, _, _, err := ParseRun([]string{b.image, "/bin/sh", "-c", args}, nil)
 	if err != nil {
 		return err
 	}
@@ -125,8 +125,8 @@ func (b *buildFile) CmdEnv(args string) error {
 	if len(tmp) != 2 {
 		return fmt.Errorf("Invalid ENV format")
 	}
-	key := strings.Trim(tmp[0], " ")
+	key := strings.Trim(tmp[0], " \t")
-	value := strings.Trim(tmp[1], " ")
+	value := strings.Trim(tmp[1], " \t")
 
 	for i, elem := range b.config.Env {
 		if strings.HasPrefix(elem, key+"=") {
@@ -165,34 +165,17 @@ func (b *buildFile) CmdCopy(args string) error {
 	return fmt.Errorf("COPY has been deprecated. Please use ADD instead")
 }
 
-func (b *buildFile) CmdAdd(args string) error {
+func (b *buildFile) addRemote(container *Container, orig, dest string) error {
-	if b.context == "" {
+	file, err := utils.Download(orig, ioutil.Discard)
-		return fmt.Errorf("No context given. Impossible to use ADD")
-	}
-	tmp := strings.SplitN(args, " ", 2)
-	if len(tmp) != 2 {
-		return fmt.Errorf("Invalid ADD format")
-	}
-	orig := strings.Trim(tmp[0], " ")
-	dest := strings.Trim(tmp[1], " ")
-
-	cmd := b.config.Cmd
-
-	// Create the container and start it
-	b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)}
-	b.config.Image = b.image
-	container, err := b.builder.Create(b.config)
 	if err != nil {
 		return err
 	}
-	b.tmpContainers[container.ID] = struct{}{}
+	defer file.Body.Close()
-	fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID))
 
-	if err := container.EnsureMounted(); err != nil {
+	return container.Inject(file.Body, dest)
-		return err
 }
-	defer container.Unmount()
 
+func (b *buildFile) addContext(container *Container, orig, dest string) error {
 	origPath := path.Join(b.context, orig)
 	destPath := path.Join(container.RootfsPath(), dest)
 	// Preserve the trailing '/'
@@ -218,6 +201,46 @@ func (b *buildFile) CmdAdd(args string) error {
 			return err
 		}
 	}
+	return nil
+}
+
+func (b *buildFile) CmdAdd(args string) error {
+	if b.context == "" {
+		return fmt.Errorf("No context given. Impossible to use ADD")
+	}
+	tmp := strings.SplitN(args, " ", 2)
+	if len(tmp) != 2 {
+		return fmt.Errorf("Invalid ADD format")
+	}
+	orig := strings.Trim(tmp[0], " \t")
+	dest := strings.Trim(tmp[1], " \t")
+
+	cmd := b.config.Cmd
+	b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)}
+
+	b.config.Image = b.image
+	// Create the container and start it
+	container, err := b.builder.Create(b.config)
+	if err != nil {
+		return err
+	}
+	b.tmpContainers[container.ID] = struct{}{}
+
+	if err := container.EnsureMounted(); err != nil {
+		return err
+	}
+	defer container.Unmount()
+
+	if utils.IsURL(orig) {
+		if err := b.addRemote(container, orig, dest); err != nil {
+			return err
+		}
+	} else {
+		if err := b.addContext(container, orig, dest); err != nil {
+			return err
+		}
+	}
+
 	if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil {
 		return err
 	}
@@ -240,7 +263,8 @@ func (b *buildFile) run() (string, error) {
 	fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID))
 
 	//start the container
-	if err := c.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := c.Start(hostConfig); err != nil {
 		return "", err
 	}
 
@@ -259,7 +283,9 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
 	}
 	b.config.Image = b.image
 	if id == "" {
+		cmd := b.config.Cmd
 		b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment}
+		defer func(cmd []string) { b.config.Cmd = cmd }(cmd)
 
 		if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil {
 			return err
@@ -271,21 +297,17 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
 		} else {
 			utils.Debugf("[BUILDER] Cache miss")
 		}
-
-		// Create the container and start it
 		container, err := b.builder.Create(b.config)
 		if err != nil {
 			return err
 		}
 		b.tmpContainers[container.ID] = struct{}{}
 		fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID))
-
+		id = container.ID
 		if err := container.EnsureMounted(); err != nil {
 			return err
 		}
 		defer container.Unmount()
-
-		id = container.ID
 	}
 
 	container := b.runtime.Get(id)
@@ -306,8 +328,9 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
 	return nil
 }
 
-func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) {
+func (b *buildFile) Build(context io.Reader) (string, error) {
-	if context != nil {
+	// FIXME: @creack any reason for using /tmp instead of ""?
+	// FIXME: @creack "name" is a terrible variable name
 	name, err := ioutil.TempDir("/tmp", "docker-build")
 	if err != nil {
 		return "", err
@@ -317,7 +340,11 @@ func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) {
 	}
 	defer os.RemoveAll(name)
 	b.context = name
+	dockerfile, err := os.Open(path.Join(name, "Dockerfile"))
+	if err != nil {
+		return "", fmt.Errorf("Can't build a directory with no Dockerfile")
 	}
+	// FIXME: "file" is also a terrible variable name ;)
 	file := bufio.NewReader(dockerfile)
 	stepN := 0
 	for {
@@ -329,7 +356,7 @@ func (b *buildFile) Build(dockerfile, context io.Reader) (string, error) {
 				return "", err
 			}
 		}
-		line = strings.Replace(strings.TrimSpace(line), "	", " ", 1)
+		line = strings.Trim(strings.Replace(line, "\t", " ", -1), " \t\r\n")
 		// Skip comments and empty line
 		if len(line) == 0 || line[0] == '#' {
 			continue

buildfile_test.go

@@ -1,89 +1,109 @@
 package docker
 
 import (
-	"github.com/dotcloud/docker/utils"
+	"io/ioutil"
-	"strings"
+	"sync"
 	"testing"
+	"fmt"
 )
 
-const Dockerfile = `
+// mkTestContext generates a build context from the contents of the provided dockerfile.
-# VERSION		0.1
+// This context is suitable for use as an argument to BuildFile.Build()
-# DOCKER-VERSION	0.2
+func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive {
+	context, err := mkBuildContext(fmt.Sprintf(dockerfile, unitTestImageId), files)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return context
+}
 
-from   ` + unitTestImageName + `
+// A testContextTemplate describes a build context and how to test it
+type testContextTemplate struct {
+	// Contents of the Dockerfile
+	dockerfile string
+	// Additional files in the context, eg [][2]string{"./passwd", "gordon"}
+	files [][2]string
+}
+
+// A table of all the contexts to build and test.
+// A new docker runtime will be created and torn down for each context.
+var testContexts []testContextTemplate = []testContextTemplate{
+	{
+		`
+from   %s
 run    sh -c 'echo root:testpass > /tmp/passwd'
 run    mkdir -p /var/run/sshd
+run    [ "$(cat /tmp/passwd)" = "root:testpass" ]
+run    [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]
+`,
+		nil,
+	},
+
+	{
 		`
+from %s
+add foo /usr/lib/bla/bar
+run [ "$(cat /usr/lib/bla/bar)" = 'hello world!' ]
+`,
+		[][2]string{{"foo", "hello world!"}},
+	},
 
-const DockerfileNoNewLine = `
+	{
-# VERSION		0.1
+		`
-# DOCKER-VERSION	0.2
+from %s
+add f /
+run [ "$(cat /f)" = "hello" ]
+add f /abc
+run [ "$(cat /abc)" = "hello" ]
+add f /x/y/z
+run [ "$(cat /x/y/z)" = "hello" ]
+add f /x/y/d/
+run [ "$(cat /x/y/d/f)" = "hello" ]
+add d /
+run [ "$(cat /ga)" = "bu" ]
+add d /somewhere
+run [ "$(cat /somewhere/ga)" = "bu" ]
+add d /anotherplace/
+run [ "$(cat /anotherplace/ga)" = "bu" ]
+add d /somewheeeere/over/the/rainbooow
+run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ]
+`,
+		[][2]string{
+			{"f", "hello"},
+			{"d/ga", "bu"},
+		},
+	},
 
-from   ` + unitTestImageName + `
+	{
-run    sh -c 'echo root:testpass > /tmp/passwd'
+		`
-run    mkdir -p /var/run/sshd`
+from %s
-
+env    FOO BAR
-// FIXME: test building with a context
+run    [ "$FOO" = "BAR" ]
-
+`,
-// FIXME: test building with a local ADD as first command
+		nil,
+	},
+}
 
 // FIXME: test building with 2 successive overlapping ADD commands
 
 func TestBuild(t *testing.T) {
-	dockerfiles := []string{Dockerfile, DockerfileNoNewLine}
+	for _, ctx := range testContexts {
-	for _, Dockerfile := range dockerfiles {
 		runtime, err := newTestRuntime()
 		if err != nil {
 			t.Fatal(err)
 		}
 		defer nuke(runtime)
 
-		srv := &Server{runtime: runtime}
+		srv := &Server{
+			runtime: runtime,
+			lock: &sync.Mutex{},
+			pullingPool: make(map[string]struct{}),
+			pushingPool: make(map[string]struct{}),
+		}
 
-		buildfile := NewBuildFile(srv, &utils.NopWriter{})
+		buildfile := NewBuildFile(srv, ioutil.Discard)
-
+		if _, err := buildfile.Build(mkTestContext(ctx.dockerfile, ctx.files, t)); err != nil {
-		imgID, err := buildfile.Build(strings.NewReader(Dockerfile), nil)
-		if err != nil {
 			t.Fatal(err)
 		}
-
-		builder := NewBuilder(runtime)
-		container, err := builder.Create(
-			&Config{
-				Image: imgID,
-				Cmd:   []string{"cat", "/tmp/passwd"},
-			},
-		)
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer runtime.Destroy(container)
-
-		output, err := container.Output()
-		if err != nil {
-			t.Fatal(err)
-		}
-		if string(output) != "root:testpass\n" {
-			t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n")
-		}
-
-		container2, err := builder.Create(
-			&Config{
-				Image: imgID,
-				Cmd:   []string{"ls", "-d", "/var/run/sshd"},
-			},
-		)
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer runtime.Destroy(container2)
-
-		output, err = container2.Output()
-		if err != nil {
-			t.Fatal(err)
-		}
-		if string(output) != "/var/run/sshd\n" {
-			t.Fatal("/var/run/sshd has not been created")
-		}
 	}
 }

commands.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
 	"flag"
@@ -10,14 +11,12 @@ import (
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
-	"mime/multipart"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"reflect"
 	"regexp"
@@ -29,7 +28,7 @@ import (
 	"unicode"
 )
 
-const VERSION = "0.4.4"
+const VERSION = "0.4.7"
 
 var (
 	GITCOMMIT string
@@ -41,7 +40,7 @@ func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) {
 }
 
 func ParseCommands(proto, addr string, args ...string) error {
-	cli := NewDockerCli(proto, addr)
+	cli := NewDockerCli(os.Stdin, os.Stdout, os.Stderr, proto, addr)
 
 	if len(args) > 0 {
 		method, exists := cli.getMethod(args[0])
@@ -65,7 +64,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 	if len(args) > 0 {
 		method, exists := cli.getMethod(args[0])
 		if !exists {
-			fmt.Println("Error: Command not found:", args[0])
+			fmt.Fprintf(cli.err, "Error: Command not found: %s\n", args[0])
 		} else {
 			method.Func.CallSlice([]reflect.Value{
 				reflect.ValueOf(cli),
@@ -75,7 +74,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 		}
 	}
 	help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n  -H=[tcp://%s:%d]: tcp://host:port to bind/connect to or unix://path/to/socker to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTHTTPHOST, DEFAULTHTTPPORT)
-	for _, command := range [][2]string{
+	for _, command := range [][]string{
 		{"attach", "Attach to a running container"},
 		{"build", "Build a container from a Dockerfile"},
 		{"commit", "Create a new image from a container's changes"},
@@ -107,7 +106,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 	} {
 		help += fmt.Sprintf("    %-10.10s%s\n", command[0], command[1])
 	}
-	fmt.Println(help)
+	fmt.Fprintf(cli.err, "%s\n", help)
 	return nil
 }
 
@@ -125,14 +124,39 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
 	v.Set("url", cmd.Arg(1))
 	v.Set("path", cmd.Arg(2))
 
-	if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, os.Stdout); err != nil {
+	if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out); err != nil {
 		return err
 	}
 	return nil
 }
 
+// mkBuildContext returns an archive of an empty context with the contents
+// of `dockerfile` at the path ./Dockerfile
+func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) {
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	files = append(files, [2]string{"Dockerfile", dockerfile})
+	for _, file := range files {
+		name, content := file[0], file[1]
+		hdr := &tar.Header{
+			Name: name,
+			Size: int64(len(content)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return nil, err
+		}
+		if _, err := tw.Write([]byte(content)); err != nil {
+			return nil, err
+		}
+	}
+	if err := tw.Close(); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
 func (cli *DockerCli) CmdBuild(args ...string) error {
-	cmd := Subcmd("build", "[OPTIONS] PATH | -", "Build a new container image from the source code at PATH")
+	cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH")
 	tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -143,68 +167,43 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	}
 
 	var (
-		multipartBody io.Reader
+		context  Archive
-		file          io.ReadCloser
+		isRemote bool
-		contextPath   string
+		err      error
 	)
 
-	// Init the needed component for the Multipart
-	buff := bytes.NewBuffer([]byte{})
-	multipartBody = buff
-	w := multipart.NewWriter(buff)
-	boundary := strings.NewReader("\r\n--" + w.Boundary() + "--\r\n")
-
-	compression := Bzip2
-
 	if cmd.Arg(0) == "-" {
-		file = os.Stdin
+		// As a special case, 'docker build -' will build from an empty context with the
+		// contents of stdin as a Dockerfile
+		dockerfile, err := ioutil.ReadAll(cli.in)
+		if err != nil {
+			return err
+		}
+		context, err = mkBuildContext(string(dockerfile), nil)
+	} else if utils.IsURL(cmd.Arg(0)) || utils.IsGIT(cmd.Arg(0)) {
+		isRemote = true
 	} else {
-		// Send Dockerfile from arg/Dockerfile (deprecate later)
+		context, err = Tar(cmd.Arg(0), Uncompressed)
-		f, err := os.Open(path.Join(cmd.Arg(0), "Dockerfile"))
-		if err != nil {
-			return err
 	}
-		file = f
+	var body io.Reader
-		// Send context from arg
+	// Setup an upload progress bar
-		// Create a FormFile multipart for the context if needed
+	// FIXME: ProgressReader shouldn't be this annoyning to use
-		// FIXME: Use NewTempArchive in order to have the size and avoid too much memory usage?
+	if context != nil {
-		context, err := Tar(cmd.Arg(0), compression)
-		if err != nil {
-			return err
-		}
-		// NOTE: Do this in case '.' or '..' is input
-		absPath, err := filepath.Abs(cmd.Arg(0))
-		if err != nil {
-			return err
-		}
-		wField, err := w.CreateFormFile("Context", filepath.Base(absPath)+"."+compression.Extension())
-		if err != nil {
-			return err
-		}
-		// FIXME: Find a way to have a progressbar for the upload too
 		sf := utils.NewStreamFormatter(false)
-		io.Copy(wField, utils.ProgressReader(ioutil.NopCloser(context), -1, os.Stdout, sf.FormatProgress("Caching Context", "%v/%v (%v)"), sf))
+		body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf)
-		multipartBody = io.MultiReader(multipartBody, boundary)
 	}
-	// Create a FormFile multipart for the Dockerfile
+	// Upload the build context
-	wField, err := w.CreateFormFile("Dockerfile", "Dockerfile")
-	if err != nil {
-		return err
-	}
-	io.Copy(wField, file)
-	multipartBody = io.MultiReader(multipartBody, boundary)
-
 	v := &url.Values{}
 	v.Set("t", *tag)
-	// Send the multipart request with correct content-type
+	if isRemote {
-	req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), multipartBody)
+		v.Set("remote", cmd.Arg(0))
+	}
+	req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body)
 	if err != nil {
 		return err
 	}
-	req.Header.Set("Content-Type", w.FormDataContentType())
+	if context != nil {
-	if contextPath != "" {
+		req.Header.Set("Content-Type", "application/tar")
-		req.Header.Set("X-Docker-Context-Compression", compression.Flag())
-		fmt.Println("Uploading Context...")
 	}
 	dial, err := net.Dial(cli.proto, cli.addr)
 	if err != nil {
@@ -217,7 +216,6 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 		return err
 	}
 	defer resp.Body.Close()
-
 	// Check for errors
 	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
 		body, err := ioutil.ReadAll(resp.Body)
@@ -231,7 +229,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	}
 
 	// Output the result
-	if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
+	if _, err := io.Copy(cli.out, resp.Body); err != nil {
 		return err
 	}
 
@@ -290,22 +288,25 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 	if err != nil {
 		return nil
 	}
+
 	var oldState *term.State
 	if *flUsername == "" || *flPassword == "" || *flEmail == "" {
-		oldState, err = term.SetRawTerminal()
+		oldState, err = term.SetRawTerminal(cli.terminalFd)
 		if err != nil {
 			return err
 		}
-		defer term.RestoreTerminal(oldState)
+		defer term.RestoreTerminal(cli.terminalFd, oldState)
 	}
 
-	var username string
+	var (
-	var password string
+		username string
-	var email string
+		password string
+		email    string
+	)
 
 	if *flUsername == "" {
-		fmt.Print("Username (", cli.authConfig.Username, "): ")
+		fmt.Fprintf(cli.out, "Username (%s): ", cli.authConfig.Username)
-		username = readAndEchoString(os.Stdin, os.Stdout)
+		username = readAndEchoString(cli.in, cli.out)
 		if username == "" {
 			username = cli.authConfig.Username
 		}
@@ -314,8 +315,8 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 	}
 	if username != cli.authConfig.Username {
 		if *flPassword == "" {
-			fmt.Print("Password: ")
+			fmt.Fprintf(cli.out, "Password: ")
-			password = readString(os.Stdin, os.Stdout)
+			password = readString(cli.in, cli.out)
 			if password == "" {
 				return fmt.Errorf("Error : Password Required")
 			}
@@ -324,8 +325,8 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 		}
 
 		if *flEmail == "" {
-			fmt.Print("Email (", cli.authConfig.Email, "): ")
+			fmt.Fprintf(cli.out, "Email (%s): ", cli.authConfig.Email)
-			email = readAndEchoString(os.Stdin, os.Stdout)
+			email = readAndEchoString(cli.in, cli.out)
 			if email == "" {
 				email = cli.authConfig.Email
 			}
@@ -337,7 +338,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 		email = cli.authConfig.Email
 	}
 	if oldState != nil {
-		term.RestoreTerminal(oldState)
+		term.RestoreTerminal(cli.terminalFd, oldState)
 	}
 	cli.authConfig.Username = username
 	cli.authConfig.Password = password
@@ -363,7 +364,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 	}
 	auth.SaveConfig(cli.authConfig)
 	if out2.Status != "" {
-		fmt.Println(out2.Status)
+		fmt.Fprintf(cli.out, "%s\n", out2.Status)
 	}
 	return nil
 }
@@ -381,14 +382,14 @@ func (cli *DockerCli) CmdWait(args ...string) error {
 	for _, name := range cmd.Args() {
 		body, _, err := cli.call("POST", "/containers/"+name+"/wait", nil)
 		if err != nil {
-			fmt.Printf("%s", err)
+			fmt.Fprintf(cli.err, "%s", err)
 		} else {
 			var out APIWait
 			err = json.Unmarshal(body, &out)
 			if err != nil {
 				return err
 			}
-			fmt.Println(out.StatusCode)
+			fmt.Fprintf(cli.out, "%d\n", out.StatusCode)
 		}
 	}
 	return nil
@@ -417,13 +418,13 @@ func (cli *DockerCli) CmdVersion(args ...string) error {
 		utils.Debugf("Error unmarshal: body: %s, err: %s\n", body, err)
 		return err
 	}
-	fmt.Println("Client version:", VERSION)
+	fmt.Fprintf(cli.out, "Client version: %s\n", VERSION)
-	fmt.Println("Server version:", out.Version)
+	fmt.Fprintf(cli.out, "Server version: %s\n", out.Version)
 	if out.GitCommit != "" {
-		fmt.Println("Git commit:", out.GitCommit)
+		fmt.Fprintf(cli.out, "Git commit: %s\n", out.GitCommit)
 	}
 	if out.GoVersion != "" {
-		fmt.Println("Go version:", out.GoVersion)
+		fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion)
 	}
 	return nil
 }
@@ -449,19 +450,19 @@ func (cli *DockerCli) CmdInfo(args ...string) error {
 		return err
 	}
 
-	fmt.Printf("Containers: %d\n", out.Containers)
+	fmt.Fprintf(cli.out, "Containers: %d\n", out.Containers)
-	fmt.Printf("Images: %d\n", out.Images)
+	fmt.Fprintf(cli.out, "Images: %d\n", out.Images)
 	if out.Debug || os.Getenv("DEBUG") != "" {
-		fmt.Printf("Debug mode (server): %v\n", out.Debug)
+		fmt.Fprintf(cli.out, "Debug mode (server): %v\n", out.Debug)
-		fmt.Printf("Debug mode (client): %v\n", os.Getenv("DEBUG") != "")
+		fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "")
-		fmt.Printf("Fds: %d\n", out.NFd)
+		fmt.Fprintf(cli.out, "Fds: %d\n", out.NFd)
-		fmt.Printf("Goroutines: %d\n", out.NGoroutines)
+		fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines)
 	}
 	if !out.MemoryLimit {
-		fmt.Println("WARNING: No memory limit support")
+		fmt.Fprintf(cli.err, "WARNING: No memory limit support\n")
 	}
 	if !out.SwapLimit {
-		fmt.Println("WARNING: No swap limit support")
+		fmt.Fprintf(cli.err, "WARNING: No swap limit support\n")
 	}
 	return nil
 }
@@ -483,9 +484,9 @@ func (cli *DockerCli) CmdStop(args ...string) error {
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("POST", "/containers/"+name+"/stop?"+v.Encode(), nil)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		} else {
-			fmt.Println(name)
+			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
 	return nil
@@ -508,9 +509,9 @@ func (cli *DockerCli) CmdRestart(args ...string) error {
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("POST", "/containers/"+name+"/restart?"+v.Encode(), nil)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		} else {
-			fmt.Println(name)
+			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
 	return nil
@@ -529,9 +530,9 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 	for _, name := range args {
 		_, _, err := cli.call("POST", "/containers/"+name+"/start", nil)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		} else {
-			fmt.Println(name)
+			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
 	return nil
@@ -546,30 +547,30 @@ func (cli *DockerCli) CmdInspect(args ...string) error {
 		cmd.Usage()
 		return nil
 	}
-	fmt.Printf("[")
+	fmt.Fprintf(cli.out, "[")
 	for i, name := range args {
 		if i > 0 {
-			fmt.Printf(",")
+			fmt.Fprintf(cli.out, ",")
 		}
 		obj, _, err := cli.call("GET", "/containers/"+name+"/json", nil)
 		if err != nil {
 			obj, _, err = cli.call("GET", "/images/"+name+"/json", nil)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "%s", err)
+				fmt.Fprintf(cli.err, "%s\n", err)
 				continue
 			}
 		}
 
 		indented := new(bytes.Buffer)
 		if err = json.Indent(indented, obj, "", "    "); err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 			continue
 		}
-		if _, err := io.Copy(os.Stdout, indented); err != nil {
+		if _, err := io.Copy(cli.out, indented); err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		}
 	}
-	fmt.Printf("]")
+	fmt.Fprintf(cli.out, "]")
 	return nil
 }
 
@@ -594,7 +595,7 @@ func (cli *DockerCli) CmdPort(args ...string) error {
 	}
 
 	if frontend, exists := out.NetworkSettings.PortMapping[cmd.Arg(1)]; exists {
-		fmt.Println(frontend)
+		fmt.Fprintf(cli.out, "%s\n", frontend)
 	} else {
 		return fmt.Errorf("Error: No private port '%s' allocated on %s", cmd.Arg(1), cmd.Arg(0))
 	}
@@ -615,7 +616,7 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 	for _, name := range cmd.Args() {
 		body, _, err := cli.call("DELETE", "/images/"+name, nil)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s", err)
+			fmt.Fprintf(cli.err, "%s", err)
 		} else {
 			var outs []APIRmi
 			err = json.Unmarshal(body, &outs)
@@ -624,9 +625,9 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 			}
 			for _, out := range outs {
 				if out.Deleted != "" {
-					fmt.Println("Deleted:", out.Deleted)
+					fmt.Fprintf(cli.out, "Deleted: %s\n", out.Deleted)
 				} else {
-					fmt.Println("Untagged:", out.Untagged)
+					fmt.Fprintf(cli.out, "Untagged: %s\n", out.Untagged)
 				}
 			}
 		}
@@ -654,7 +655,7 @@ func (cli *DockerCli) CmdHistory(args ...string) error {
 	if err != nil {
 		return err
 	}
-	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
+	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 	fmt.Fprintln(w, "ID\tCREATED\tCREATED BY")
 
 	for _, out := range outs {
@@ -684,9 +685,9 @@ func (cli *DockerCli) CmdRm(args ...string) error {
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("DELETE", "/containers/"+name+"?"+val.Encode(), nil)
 		if err != nil {
-			fmt.Printf("%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		} else {
-			fmt.Println(name)
+			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
 	return nil
@@ -706,9 +707,9 @@ func (cli *DockerCli) CmdKill(args ...string) error {
 	for _, name := range args {
 		_, _, err := cli.call("POST", "/containers/"+name+"/kill", nil)
 		if err != nil {
-			fmt.Printf("%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
 		} else {
-			fmt.Println(name)
+			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
 	return nil
@@ -730,7 +731,7 @@ func (cli *DockerCli) CmdImport(args ...string) error {
 	v.Set("tag", tag)
 	v.Set("fromSrc", src)
 
-	err := cli.stream("POST", "/images/create?"+v.Encode(), os.Stdin, os.Stdout)
+	err := cli.stream("POST", "/images/create?"+v.Encode(), cli.in, cli.out)
 	if err != nil {
 		return err
 	}
@@ -754,14 +755,15 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 		return err
 	}
 
+	if *registry == "" {
+		// If we're not using a custom registry, we know the restrictions
+		// applied to repository names and can warn the user in advance.
+		// Custom repositories can have different rules, and we must also
+		// allow pushing by image ID.
 		if len(strings.SplitN(name, "/", 2)) == 1 {
 			return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in <user>/<repo> (ex: %s/%s)", cli.authConfig.Username, name)
 		}
 
-	buf, err := json.Marshal(cli.authConfig)
-	if err != nil {
-		return err
-	}
 		nameParts := strings.SplitN(name, "/", 2)
 		validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
 		if !validNamespace.MatchString(nameParts[0]) {
@@ -771,10 +773,16 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 		if !validRepo.MatchString(nameParts[1]) {
 			return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1])
 		}
+	}
+
+	buf, err := json.Marshal(cli.authConfig)
+	if err != nil {
+		return err
+	}
 
 	v := url.Values{}
 	v.Set("registry", *registry)
-	if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), os.Stdout); err != nil {
+	if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil {
 		return err
 	}
 	return nil
@@ -805,7 +813,7 @@ func (cli *DockerCli) CmdPull(args ...string) error {
 	v.Set("tag", *tag)
 	v.Set("registry", *registry)
 
-	if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, os.Stdout); err != nil {
+	if err := cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out); err != nil {
 		return err
 	}
 
@@ -832,7 +840,7 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s", body)
+		fmt.Fprintf(cli.out, "%s", body)
 	} else {
 		v := url.Values{}
 		if cmd.NArg() == 1 {
@@ -853,7 +861,7 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 			return err
 		}
 
-		w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
+		w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 		if !*quiet {
 			fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED\tSIZE")
 		}
@@ -898,6 +906,7 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 func (cli *DockerCli) CmdPs(args ...string) error {
 	cmd := Subcmd("ps", "[OPTIONS]", "List containers")
 	quiet := cmd.Bool("q", false, "Only display numeric IDs")
+	size := cmd.Bool("s", false, "Display sizes")
 	all := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.")
 	noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
 	nLatest := cmd.Bool("l", false, "Show only the latest created container, include non-running ones.")
@@ -924,6 +933,9 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 	if *before != "" {
 		v.Set("before", *before)
 	}
+	if *size {
+		v.Set("size", "1")
+	}
 
 	body, _, err := cli.call("GET", "/containers/json?"+v.Encode(), nil)
 	if err != nil {
@@ -935,9 +947,14 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 	if err != nil {
 		return err
 	}
-	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
+	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 	if !*quiet {
-		fmt.Fprintln(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS\tSIZE")
+		fmt.Fprint(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS")
+		if *size {
+			fmt.Fprintln(w, "\tSIZE")
+		} else {
+			fmt.Fprint(w, "\n")
+		}
 	}
 
 	for _, out := range outs {
@@ -947,11 +964,15 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 			} else {
 				fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\t%s\t", utils.TruncateID(out.ID), out.Image, utils.Trunc(out.Command, 20), utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Status, out.Ports)
 			}
+			if *size {
 				if out.SizeRootFs > 0 {
 					fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.SizeRw), utils.HumanSize(out.SizeRootFs))
 				} else {
 					fmt.Fprintf(w, "%s\n", utils.HumanSize(out.SizeRw))
 				}
+			} else {
+				fmt.Fprint(w, "\n")
+			}
 		} else {
 			if *noTrunc {
 				fmt.Fprintln(w, out.ID)
@@ -1005,7 +1026,7 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 		return err
 	}
 
-	fmt.Println(apiID.ID)
+	fmt.Fprintf(cli.out, "%s\n", apiID.ID)
 	return nil
 }
 
@@ -1020,7 +1041,7 @@ func (cli *DockerCli) CmdExport(args ...string) error {
 		return nil
 	}
 
-	if err := cli.stream("GET", "/containers/"+cmd.Arg(0)+"/export", nil, os.Stdout); err != nil {
+	if err := cli.stream("GET", "/containers/"+cmd.Arg(0)+"/export", nil, cli.out); err != nil {
 		return err
 	}
 	return nil
@@ -1047,7 +1068,7 @@ func (cli *DockerCli) CmdDiff(args ...string) error {
 		return err
 	}
 	for _, change := range changes {
-		fmt.Println(change.String())
+		fmt.Fprintf(cli.out, "%s\n", change.String())
 	}
 	return nil
 }
@@ -1062,10 +1083,10 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
 		return nil
 	}
 
-	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", false, nil, os.Stdout); err != nil {
+	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1", false, nil, cli.out); err != nil {
 		return err
 	}
-	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", false, nil, os.Stderr); err != nil {
+	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stderr=1", false, nil, cli.err); err != nil {
 		return err
 	}
 	return nil
@@ -1097,7 +1118,9 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 	}
 
 	if container.Config.Tty {
-		cli.monitorTtySize(cmd.Arg(0))
+		if err := cli.monitorTtySize(cmd.Arg(0)); err != nil {
+			return err
+		}
 	}
 
 	v := url.Values{}
@@ -1106,7 +1129,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 	v.Set("stdout", "1")
 	v.Set("stderr", "1")
 
-	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, os.Stdin, os.Stdout); err != nil {
+	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, cli.in, cli.out); err != nil {
 		return err
 	}
 	return nil
@@ -1134,8 +1157,8 @@ func (cli *DockerCli) CmdSearch(args ...string) error {
 	if err != nil {
 		return err
 	}
-	fmt.Printf("Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0))
+	fmt.Fprintf(cli.out, "Found %d results matching your query (\"%s\")\n", len(outs), cmd.Arg(0))
-	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
+	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 	fmt.Fprintf(w, "NAME\tDESCRIPTION\n")
 	for _, out := range outs {
 		desc := strings.Replace(out.Description, "\n", " ", -1)
@@ -1238,7 +1261,7 @@ func (cli *DockerCli) CmdTag(args ...string) error {
 }
 
 func (cli *DockerCli) CmdRun(args ...string) error {
-	config, cmd, err := ParseRun(args, nil)
+	config, hostConfig, cmd, err := ParseRun(args, nil)
 	if err != nil {
 		return err
 	}
@@ -1253,7 +1276,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	if statusCode == 404 {
 		v := url.Values{}
 		v.Set("fromImage", config.Image)
-		err = cli.stream("POST", "/images/create?"+v.Encode(), nil, os.Stderr)
+		err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err)
 		if err != nil {
 			return err
 		}
@@ -1266,27 +1289,31 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		return err
 	}
 
-	out := &APIRun{}
+	runResult := &APIRun{}
-	err = json.Unmarshal(body, out)
+	err = json.Unmarshal(body, runResult)
 	if err != nil {
 		return err
 	}
 
-	for _, warning := range out.Warnings {
+	for _, warning := range runResult.Warnings {
-		fmt.Fprintln(os.Stderr, "WARNING: ", warning)
+		fmt.Fprintln(cli.err, "WARNING: ", warning)
 	}
 
 	//start the container
-	_, _, err = cli.call("POST", "/containers/"+out.ID+"/start", nil)
+	if _, _, err = cli.call("POST", "/containers/"+runResult.ID+"/start", hostConfig); err != nil {
-	if err != nil {
 		return err
 	}
 
 	if !config.AttachStdout && !config.AttachStderr {
-		fmt.Println(out.ID)
+		// Make this asynchrone in order to let the client write to stdin before having to read the ID
-	} else {
+		go fmt.Fprintf(cli.out, "%s\n", runResult.ID)
+	}
+
+	if config.AttachStdin || config.AttachStdout || config.AttachStderr {
 		if config.Tty {
-			cli.monitorTtySize(out.ID)
+			if err := cli.monitorTtySize(runResult.ID); err != nil {
+				return err
+			}
 		}
 
 		v := url.Values{}
@@ -1302,7 +1329,8 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		if config.AttachStderr {
 			v.Set("stderr", "1")
 		}
-		if err := cli.hijack("POST", "/containers/"+out.ID+"/attach?"+v.Encode(), config.Tty, os.Stdin, os.Stdout); err != nil {
+
+		if err := cli.hijack("POST", "/containers/"+runResult.ID+"/attach?"+v.Encode(), config.Tty, cli.in, cli.out); err != nil {
 			utils.Debugf("Error hijack: %s", err)
 			return err
 		}
@@ -1433,7 +1461,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e
 	return nil
 }
 
-func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.File, out io.Writer) error {
+func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, out io.Writer) error {
 
 	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil)
 	if err != nil {
@@ -1460,18 +1488,27 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi
 		return err
 	})
 
-	if in != nil && setRawTerminal && term.IsTerminal(in.Fd()) && os.Getenv("NORAW") == "" {
+	if in != nil && setRawTerminal && cli.isTerminal && os.Getenv("NORAW") == "" {
-		oldState, err := term.SetRawTerminal()
+		oldState, err := term.SetRawTerminal(cli.terminalFd)
 		if err != nil {
 			return err
 		}
-		defer term.RestoreTerminal(oldState)
+		defer term.RestoreTerminal(cli.terminalFd, oldState)
 	}
+
 	sendStdin := utils.Go(func() error {
+		if in != nil {
 			io.Copy(rwc, in)
-		if err := rwc.(*net.TCPConn).CloseWrite(); err != nil {
+		}
+		if tcpc, ok := rwc.(*net.TCPConn); ok {
+			if err := tcpc.CloseWrite(); err != nil {
 				utils.Debugf("Couldn't send EOF: %s\n", err)
 			}
+		} else if unixc, ok := rwc.(*net.UnixConn); ok {
+			if err := unixc.CloseWrite(); err != nil {
+				utils.Debugf("Couldn't send EOF: %s\n", err)
+			}
+		}
 		// Discard errors due to pipe interruption
 		return nil
 	})
@@ -1481,7 +1518,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi
 		return err
 	}
 
-	if !term.IsTerminal(in.Fd()) {
+	if !cli.isTerminal {
 		if err := <-sendStdin; err != nil {
 			utils.Debugf("Error sendStdin: %s", err)
 			return err
@@ -1492,7 +1529,10 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in *os.Fi
 }
 
 func (cli *DockerCli) resizeTty(id string) {
-	ws, err := term.GetWinsize(os.Stdin.Fd())
+	if !cli.isTerminal {
+		return
+	}
+	ws, err := term.GetWinsize(cli.terminalFd)
 	if err != nil {
 		utils.Debugf("Error getting size: %s", err)
 	}
@@ -1504,7 +1544,10 @@ func (cli *DockerCli) resizeTty(id string) {
 	}
 }
 
-func (cli *DockerCli) monitorTtySize(id string) {
+func (cli *DockerCli) monitorTtySize(id string) error {
+	if !cli.isTerminal {
+		return fmt.Errorf("Impossible to monitor size on non-tty")
+	}
 	cli.resizeTty(id)
 
 	c := make(chan os.Signal, 1)
@@ -1516,24 +1559,56 @@ func (cli *DockerCli) monitorTtySize(id string) {
 			}
 		}
 	}()
+	return nil
 }
 
 func Subcmd(name, signature, description string) *flag.FlagSet {
 	flags := flag.NewFlagSet(name, flag.ContinueOnError)
 	flags.Usage = func() {
-		fmt.Printf("\nUsage: docker %s %s\n\n%s\n\n", name, signature, description)
+		// FIXME: use custom stdout or return error
+		fmt.Fprintf(os.Stdout, "\nUsage: docker %s %s\n\n%s\n\n", name, signature, description)
 		flags.PrintDefaults()
 	}
 	return flags
 }
 
-func NewDockerCli(proto, addr string) *DockerCli {
+func NewDockerCli(in io.ReadCloser, out, err io.Writer, proto, addr string) *DockerCli {
+	var (
+		isTerminal bool = false
+		terminalFd uintptr
+	)
+
+	if in != nil {
+		if file, ok := in.(*os.File); ok {
+			terminalFd = file.Fd()
+			isTerminal = term.IsTerminal(terminalFd)
+		}
+	}
+
+	if err == nil {
+		err = out
+	}
+
 	authConfig, _ := auth.LoadConfig(os.Getenv("HOME"))
-	return &DockerCli{proto, addr, authConfig}
+	return &DockerCli{
+		proto:      proto,
+		addr:       addr,
+		authConfig: authConfig,
+		in:         in,
+		out:        out,
+		err:        err,
+		isTerminal: isTerminal,
+		terminalFd: terminalFd,
+	}
 }
 
 type DockerCli struct {
 	proto      string
 	addr       string
 	authConfig *auth.AuthConfig
+	in         io.ReadCloser
+	out        io.Writer
+	err        io.Writer
+	isTerminal bool
+	terminalFd uintptr
 }

commands_test.go

@@ -3,8 +3,9 @@ package docker
 import (
 	"bufio"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
-	_ "io/ioutil"
+	"io/ioutil"
 	"strings"
 	"testing"
 	"time"
@@ -59,20 +60,6 @@ func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error
 }
 
 /*TODO
-func cmdWait(srv *Server, container *Container) error {
-	stdout, stdoutPipe := io.Pipe()
-
-	go func() {
-		srv.CmdWait(nil, stdoutPipe, container.Id)
-	}()
-
-	if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil {
-		return err
-	}
-	// Cleanup pipes
-	return closeWrap(stdout, stdoutPipe)
-}
-
 func cmdImages(srv *Server, args ...string) (string, error) {
 	stdout, stdoutPipe := io.Pipe()
 
@@ -144,26 +131,23 @@ func TestImages(t *testing.T) {
 	// todo: add checks for -a
 }
 
+*/
 // TestRunHostname checks that 'docker run -h' correctly sets a custom hostname
 func TestRunHostname(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer nuke(runtime)
-
-	srv := &Server{runtime: runtime}
-
-	stdin, _ := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
+	cli := NewDockerCli(nil, stdoutPipe, nil, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
 	c := make(chan struct{})
 	go func() {
-		if err := srv.CmdRun(stdin, rcli.NewDockerLocalConn(stdoutPipe), "-h", "foobar", GetTestImage(runtime).Id, "hostname"); err != nil {
+		defer close(c)
+		if err := cli.CmdRun("-h", "foobar", unitTestImageId, "hostname"); err != nil {
 			t.Fatal(err)
 		}
-		close(c)
 	}()
+	utils.Debugf("--")
+	setTimeout(t, "Reading command output time out", 2*time.Second, func() {
 		cmdOutput, err := bufio.NewReader(stdout).ReadString('\n')
 		if err != nil {
 			t.Fatal(err)
@@ -171,14 +155,15 @@ func TestRunHostname(t *testing.T) {
 		if cmdOutput != "foobar\n" {
 			t.Fatalf("'hostname' should display '%s', not '%s'", "foobar\n", cmdOutput)
 		}
+	})
 
-	setTimeout(t, "CmdRun timed out", 2*time.Second, func() {
+	setTimeout(t, "CmdRun timed out", 5*time.Second, func() {
 		<-c
-		cmdWait(srv, srv.runtime.List()[0])
 	})
 
 }
 
+/*
 func TestRunExit(t *testing.T) {
 	runtime, err := newTestRuntime()
 	if err != nil {
@@ -334,29 +319,27 @@ func TestRunDisconnectTty(t *testing.T) {
 		t.Fatalf("/bin/cat should  still be running after closing stdin (tty mode)")
 	}
 }
+*/
 
 // TestAttachStdin checks attaching to stdin without stdout and stderr.
 // 'docker run -i -a stdin' should sends the client's stdin to the command,
 // then detach from it and print the container id.
 func TestRunAttachStdin(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer nuke(runtime)
-	srv := &Server{runtime: runtime}
 
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
+	cli := NewDockerCli(stdin, stdoutPipe, nil, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
 	ch := make(chan struct{})
 	go func() {
-		srv.CmdRun(stdin, rcli.NewDockerLocalConn(stdoutPipe), "-i", "-a", "stdin", GetTestImage(runtime).Id, "sh", "-c", "echo hello; cat")
+		defer close(ch)
-		close(ch)
+		cli.CmdRun("-i", "-a", "stdin", unitTestImageId, "sh", "-c", "echo hello && cat")
 	}()
 
 	// Send input to the command, close stdin
-	setTimeout(t, "Write timed out", 2*time.Second, func() {
+	setTimeout(t, "Write timed out", 10*time.Second, func() {
 		if _, err := stdinPipe.Write([]byte("hi there\n")); err != nil {
 			t.Fatal(err)
 		}
@@ -365,23 +348,27 @@ func TestRunAttachStdin(t *testing.T) {
 		}
 	})
 
-	container := runtime.List()[0]
+	container := globalRuntime.List()[0]
 
 	// Check output
+	setTimeout(t, "Reading command output time out", 10*time.Second, func() {
 		cmdOutput, err := bufio.NewReader(stdout).ReadString('\n')
 		if err != nil {
 			t.Fatal(err)
 		}
-	if cmdOutput != container.ShortId()+"\n" {
+		if cmdOutput != container.ShortID()+"\n" {
-		t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortId()+"\n", cmdOutput)
+			t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortID()+"\n", cmdOutput)
 		}
+	})
 
 	// wait for CmdRun to return
-	setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() {
+	setTimeout(t, "Waiting for CmdRun timed out", 5*time.Second, func() {
+		// Unblock hijack end
+		stdout.Read([]byte{})
 		<-ch
 	})
 
-	setTimeout(t, "Waiting for command to exit timed out", 2*time.Second, func() {
+	setTimeout(t, "Waiting for command to exit timed out", 5*time.Second, func() {
 		container.Wait()
 	})
 
@@ -400,6 +387,7 @@ func TestRunAttachStdin(t *testing.T) {
 	}
 }
 
+/*
 // Expected behaviour, the process stays alive when the client disconnects
 func TestAttachDisconnect(t *testing.T) {
 	runtime, err := newTestRuntime()

container.go

@@ -52,6 +52,9 @@ type Container struct {
 
 	waitLock chan struct{}
 	Volumes  map[string]string
+	// Store rw/ro in a separate structure to preserve reserve-compatibility on-disk.
+	// Easier than migrating older container configs :)
+	VolumesRW map[string]bool
 }
 
 type Config struct {
@@ -75,8 +78,18 @@ type Config struct {
 	VolumesFrom  string
 }
 
-func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet, error) {
+type HostConfig struct {
-	cmd := Subcmd("run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container")
+	Binds []string
+}
+
+type BindMap struct {
+	SrcPath string
+	DstPath string
+	Mode    string
+}
+
+func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
+	cmd := Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container")
 	if len(args) > 0 && args[0] != "--help" {
 		cmd.SetOutput(ioutil.Discard)
 	}
@@ -111,11 +124,14 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet
 
 	flVolumesFrom := cmd.String("volumes-from", "", "Mount volumes from the specified container")
 
+	var flBinds ListOpts
+	cmd.Var(&flBinds, "b", "Bind mount a volume from the host (e.g. -b /host:/container)")
+
 	if err := cmd.Parse(args); err != nil {
-		return nil, cmd, err
+		return nil, nil, cmd, err
 	}
 	if *flDetach && len(flAttach) > 0 {
-		return nil, cmd, fmt.Errorf("Conflicting options: -a and -d")
+		return nil, nil, cmd, fmt.Errorf("Conflicting options: -a and -d")
 	}
 	// If neither -d or -a are set, attach to everything by default
 	if len(flAttach) == 0 && !*flDetach {
@@ -127,6 +143,14 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet
 			}
 		}
 	}
+
+	// add any bind targets to the list of container volumes
+	for _, bind := range flBinds {
+		arr := strings.Split(bind, ":")
+		dstDir := arr[1]
+		flVolumes[dstDir] = struct{}{}
+	}
+
 	parsedArgs := cmd.Args()
 	runCmd := []string{}
 	image := ""
@@ -154,6 +178,9 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet
 		Volumes:      flVolumes,
 		VolumesFrom:  *flVolumesFrom,
 	}
+	hostConfig := &HostConfig{
+		Binds: flBinds,
+	}
 
 	if capabilities != nil && *flMemory > 0 && !capabilities.SwapLimit {
 		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n")
@@ -164,7 +191,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *flag.FlagSet
 	if config.OpenStdin && config.AttachStdin {
 		config.StdinOnce = true
 	}
-	return config, cmd, nil
+	return config, hostConfig, cmd, nil
 }
 
 type NetworkSettings struct {
@@ -430,7 +457,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 	})
 }
 
-func (container *Container) Start() error {
+func (container *Container) Start(hostConfig *HostConfig) error {
 	container.State.lock()
 	defer container.State.unlock()
 
@@ -454,17 +481,71 @@ func (container *Container) Start() error {
 		container.Config.MemorySwap = -1
 	}
 	container.Volumes = make(map[string]string)
+	container.VolumesRW = make(map[string]bool)
 
+	// Create the requested bind mounts
+	binds := make(map[string]BindMap)
+	// Define illegal container destinations
+	illegal_dsts := []string{"/", "."}
+
+	for _, bind := range hostConfig.Binds {
+		// FIXME: factorize bind parsing in parseBind
+		var src, dst, mode string
+		arr := strings.Split(bind, ":")
+		if len(arr) == 2 {
+			src = arr[0]
+			dst = arr[1]
+			mode = "rw"
+		} else if len(arr) == 3 {
+			src = arr[0]
+			dst = arr[1]
+			mode = arr[2]
+		} else {
+			return fmt.Errorf("Invalid bind specification: %s", bind)
+		}
+
+		// Bail if trying to mount to an illegal destination
+		for _, illegal := range illegal_dsts {
+			if dst == illegal {
+				return fmt.Errorf("Illegal bind destination: %s", dst)
+			}
+		}
+
+		bindMap := BindMap{
+			SrcPath: src,
+			DstPath: dst,
+			Mode:    mode,
+		}
+		binds[path.Clean(dst)] = bindMap
+	}
+
+	// FIXME: evaluate volumes-from before individual volumes, so that the latter can override the former.
 	// Create the requested volumes volumes
 	for volPath := range container.Config.Volumes {
+		volPath = path.Clean(volPath)
+		// If an external bind is defined for this volume, use that as a source
+		if bindMap, exists := binds[volPath]; exists {
+			container.Volumes[volPath] = bindMap.SrcPath
+			if strings.ToLower(bindMap.Mode) == "rw" {
+				container.VolumesRW[volPath] = true
+			}
+			// Otherwise create an directory in $ROOT/volumes/ and use that
+		} else {
 			c, err := container.runtime.volumes.Create(nil, container, "", "", nil)
 			if err != nil {
 				return err
 			}
+			srcPath, err := c.layer()
+			if err != nil {
+				return err
+			}
+			container.Volumes[volPath] = srcPath
+			container.VolumesRW[volPath] = true // RW by default
+		}
+		// Create the mountpoint
 		if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil {
 			return nil
 		}
-		container.Volumes[volPath] = c.ID
 	}
 
 	if container.Config.VolumesFrom != "" {
@@ -552,7 +633,8 @@ func (container *Container) Start() error {
 }
 
 func (container *Container) Run() error {
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		return err
 	}
 	container.Wait()
@@ -565,7 +647,8 @@ func (container *Container) Output() (output []byte, err error) {
 		return nil, err
 	}
 	defer pipe.Close()
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		return nil, err
 	}
 	output, err = ioutil.ReadAll(pipe)
@@ -632,7 +715,6 @@ func (container *Container) waitLxc() error {
 		}
 		time.Sleep(500 * time.Millisecond)
 	}
-	panic("Unreachable")
 }
 
 func (container *Container) monitor() {
@@ -769,7 +851,8 @@ func (container *Container) Restart(seconds int) error {
 	if err := container.Stop(seconds); err != nil {
 		return err
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		return err
 	}
 	return nil
@@ -821,8 +904,6 @@ func (container *Container) WaitTimeout(timeout time.Duration) error {
 	case <-done:
 		return nil
 	}
-
-	panic("Unreachable")
 }
 
 func (container *Container) EnsureMounted() error {
@@ -894,22 +975,6 @@ func (container *Container) RootfsPath() string {
 	return path.Join(container.root, "rootfs")
 }
 
-func (container *Container) GetVolumes() (map[string]string, error) {
-	ret := make(map[string]string)
-	for volPath, id := range container.Volumes {
-		volume, err := container.runtime.volumes.Get(id)
-		if err != nil {
-			return nil, err
-		}
-		root, err := volume.root()
-		if err != nil {
-			return nil, err
-		}
-		ret[volPath] = path.Join(root, "layer")
-	}
-	return ret, nil
-}
-
 func (container *Container) rwPath() string {
 	return path.Join(container.root, "rw")
 }

container_test.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"path"
 	"regexp"
 	"sort"
 	"strings"
@@ -15,10 +16,7 @@ import (
 )
 
 func TestIDFormat(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container1, err := NewBuilder(runtime).Create(
 		&Config{
@@ -39,10 +37,7 @@ func TestIDFormat(t *testing.T) {
 }
 
 func TestMultipleAttachRestart(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(
 		&Config{
@@ -70,7 +65,8 @@ func TestMultipleAttachRestart(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	l1, err := bufio.NewReader(stdout1).ReadString('\n')
@@ -111,7 +107,7 @@ func TestMultipleAttachRestart(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -142,10 +138,7 @@ func TestMultipleAttachRestart(t *testing.T) {
 }
 
 func TestDiff(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
@@ -251,10 +244,7 @@ func TestDiff(t *testing.T) {
 }
 
 func TestCommitAutoRun(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
@@ -306,7 +296,8 @@ func TestCommitAutoRun(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container2.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container2.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	container2.Wait()
@@ -330,10 +321,7 @@ func TestCommitAutoRun(t *testing.T) {
 }
 
 func TestCommitRun(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
@@ -388,7 +376,8 @@ func TestCommitRun(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container2.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container2.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	container2.Wait()
@@ -412,10 +401,7 @@ func TestCommitRun(t *testing.T) {
 }
 
 func TestStart(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(
 		&Config{
@@ -436,7 +422,8 @@ func TestStart(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -446,7 +433,7 @@ func TestStart(t *testing.T) {
 	if !container.State.Running {
 		t.Errorf("Container should be running")
 	}
-	if err := container.Start(); err == nil {
+	if err := container.Start(hostConfig); err == nil {
 		t.Fatalf("A running containter should be able to be started")
 	}
 
@@ -456,10 +443,7 @@ func TestStart(t *testing.T) {
 }
 
 func TestRun(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(
 		&Config{
@@ -484,10 +468,7 @@ func TestRun(t *testing.T) {
 }
 
 func TestOutput(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(
 		&Config{
@@ -509,10 +490,7 @@ func TestOutput(t *testing.T) {
 }
 
 func TestKillDifferentUser(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -528,7 +506,8 @@ func TestKillDifferentUser(t *testing.T) {
 	if container.State.Running {
 		t.Errorf("Container shouldn't be running")
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -556,15 +535,33 @@ func TestKillDifferentUser(t *testing.T) {
 	}
 }
 
-func TestKill(t *testing.T) {
+// Test that creating a container with a volume doesn't crash. Regression test for #995.
-	runtime, err := newTestRuntime()
+func TestCreateVolume(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	config, hc, _, err := ParseRun([]string{"-v", "/var/lib/data", GetTestImage(runtime).ID, "echo", "hello", "world"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	c, err := NewBuilder(runtime).Create(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(c)
+	if err := c.Start(hc); err != nil {
+		t.Fatal(err)
+	}
+	c.WaitTimeout(500 * time.Millisecond)
+	c.Wait()
+}
+
+func TestKill(t *testing.T) {
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
-		Cmd:   []string{"cat", "/dev/zero"},
+		Cmd:   []string{"sleep", "2"},
 	},
 	)
 	if err != nil {
@@ -575,7 +572,8 @@ func TestKill(t *testing.T) {
 	if container.State.Running {
 		t.Errorf("Container shouldn't be running")
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -602,10 +600,7 @@ func TestKill(t *testing.T) {
 }
 
 func TestExitCode(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
@@ -642,10 +637,7 @@ func TestExitCode(t *testing.T) {
 }
 
 func TestRestart(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -675,10 +667,7 @@ func TestRestart(t *testing.T) {
 }
 
 func TestRestartStdin(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -700,7 +689,8 @@ func TestRestartStdin(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := io.WriteString(stdin, "hello world"); err != nil {
@@ -730,7 +720,7 @@ func TestRestartStdin(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := io.WriteString(stdin, "hello world #2"); err != nil {
@@ -753,10 +743,7 @@ func TestRestartStdin(t *testing.T) {
 }
 
 func TestUser(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
@@ -863,17 +850,14 @@ func TestUser(t *testing.T) {
 }
 
 func TestMultipleContainers(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 
 	builder := NewBuilder(runtime)
 
 	container1, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).ID,
-		Cmd:   []string{"cat", "/dev/zero"},
+		Cmd:   []string{"sleep", "2"},
 	},
 	)
 	if err != nil {
@@ -883,7 +867,7 @@ func TestMultipleContainers(t *testing.T) {
 
 	container2, err := builder.Create(&Config{
 		Image: GetTestImage(runtime).ID,
-		Cmd:   []string{"cat", "/dev/zero"},
+		Cmd:   []string{"sleep", "2"},
 	},
 	)
 	if err != nil {
@@ -892,10 +876,11 @@ func TestMultipleContainers(t *testing.T) {
 	defer runtime.Destroy(container2)
 
 	// Start both containers
-	if err := container1.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container1.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
-	if err := container2.Start(); err != nil {
+	if err := container2.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 
@@ -922,10 +907,7 @@ func TestMultipleContainers(t *testing.T) {
 }
 
 func TestStdin(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -947,7 +929,8 @@ func TestStdin(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	defer stdin.Close()
@@ -969,10 +952,7 @@ func TestStdin(t *testing.T) {
 }
 
 func TestTty(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -994,7 +974,8 @@ func TestTty(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	defer stdin.Close()
@@ -1016,10 +997,7 @@ func TestTty(t *testing.T) {
 }
 
 func TestEnv(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	container, err := NewBuilder(runtime).Create(&Config{
 		Image: GetTestImage(runtime).ID,
@@ -1036,7 +1014,8 @@ func TestEnv(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer stdout.Close()
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 	container.Wait()
@@ -1085,10 +1064,7 @@ func grepFile(t *testing.T, path string, pattern string) {
 }
 
 func TestLXCConfig(t *testing.T) {
-	runtime, err := newTestRuntime()
+	runtime := mkRuntime(t)
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer nuke(runtime)
 	// Memory is allocated randomly for testing
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -1172,7 +1148,8 @@ func BenchmarkRunParallel(b *testing.B) {
 				return
 			}
 			defer runtime.Destroy(container)
-			if err := container.Start(); err != nil {
+			hostConfig := &HostConfig{}
+			if err := container.Start(hostConfig); err != nil {
 				complete <- err
 				return
 			}
@@ -1201,3 +1178,35 @@ func BenchmarkRunParallel(b *testing.B) {
 		b.Fatal(errors)
 	}
 }
+
+func tempDir(t *testing.T) string {
+	tmpDir, err := ioutil.TempDir("", "docker-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tmpDir
+}
+
+func TestBindMounts(t *testing.T) {
+	r := mkRuntime(t)
+	defer nuke(r)
+	tmpDir := tempDir(t)
+	defer os.RemoveAll(tmpDir)
+	writeFile(path.Join(tmpDir, "touch-me"), "", t)
+
+	// Test reading from a read-only bind mount
+	stdout, _ := runContainer(r, []string{"-b", fmt.Sprintf("%s:/tmp:ro", tmpDir), "_", "ls", "/tmp"}, t)
+	if !strings.Contains(stdout, "touch-me") {
+		t.Fatal("Container failed to read from bind mount")
+	}
+
+	// test writing to bind mount
+	runContainer(r, []string{"-b", fmt.Sprintf("%s:/tmp:rw", tmpDir), "_", "touch", "/tmp/holla"}, t)
+	readFile(path.Join(tmpDir, "holla"), t) // Will fail if the file doesn't exist
+
+	// test mounting to an illegal destination directory
+	if _, err := runContainer(r, []string{"-b", fmt.Sprintf("%s:.", tmpDir), "ls", "."}, nil); err == nil {
+		t.Fatal("Container bind mounted illegal directory")
+
+	}
+}

contrib/crashTest.go

@@ -116,7 +116,6 @@ func crashTest() error {
 			return err
 		}
 	}
-	return nil
 }
 
 func main() {

contrib/mkimage-unittest.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Generate a very minimal filesystem based on busybox-static,
+# and load it into the local docker under the name "docker-ut".
+
+missing_pkg() {
+    echo "Sorry, I could not locate $1"
+    echo "Try 'apt-get install ${2:-$1}'?"
+    exit 1
+}
+
+BUSYBOX=$(which busybox)
+[ "$BUSYBOX" ] || missing_pkg busybox busybox-static
+SOCAT=$(which socat)
+[ "$SOCAT" ] || missing_pkg socat
+
+shopt -s extglob
+set -ex
+ROOTFS=`mktemp -d /tmp/rootfs-busybox.XXXXXXXXXX`
+trap "rm -rf $ROOTFS" INT QUIT TERM
+cd $ROOTFS
+
+mkdir bin etc dev dev/pts lib proc sys tmp
+touch etc/resolv.conf
+cp /etc/nsswitch.conf etc/nsswitch.conf
+echo root:x:0:0:root:/:/bin/sh > etc/passwd
+echo root:x:0: > etc/group
+ln -s lib lib64
+ln -s bin sbin
+cp $BUSYBOX $SOCAT bin
+for X in $(busybox --list)
+do
+    ln -s busybox bin/$X
+done
+rm bin/init
+ln bin/busybox bin/init
+cp -P /lib/x86_64-linux-gnu/lib{pthread*,c*(-*),dl*(-*),nsl*(-*),nss_*,util*(-*),wrap,z}.so* lib
+cp /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 lib
+cp -P /usr/lib/x86_64-linux-gnu/lib{crypto,ssl}.so* lib
+for X in console null ptmx random stdin stdout stderr tty urandom zero
+do
+    cp -a /dev/$X dev
+done
+
+tar -cf- . | docker import - docker-ut
+docker run -i -u root docker-ut /bin/echo Success.
+rm -rf $ROOTFS

docker/docker.go

@@ -30,6 +30,7 @@ func main() {
 	flAutoRestart := flag.Bool("r", false, "Restart previously running containers")
 	bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge")
 	pidfile := flag.String("p", "/var/run/docker.pid", "File containing process PID")
+	flGraphPath := flag.String("g", "/var/lib/docker", "Path to graph storage base dir.")
 	flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS requests in the remote api.")
 	flDns := flag.String("dns", "", "Set custom dns servers")
 	flHosts := docker.ListOpts{fmt.Sprintf("tcp://%s:%d", docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT)}
@@ -56,7 +57,7 @@ func main() {
 			flag.Usage()
 			return
 		}
-		if err := daemon(*pidfile, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil {
+		if err := daemon(*pidfile, *flGraphPath, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil {
 			log.Fatal(err)
 			os.Exit(-1)
 		}
@@ -100,7 +101,7 @@ func removePidFile(pidfile string) {
 	}
 }
 
-func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error {
+func daemon(pidfile string, flGraphPath string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error {
 	if err := createPidFile(pidfile); err != nil {
 		log.Fatal(err)
 	}
@@ -118,7 +119,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
 	if flDns != "" {
 		dns = []string{flDns}
 	}
-	server, err := docker.NewServer(autoRestart, enableCors, dns)
+	server, err := docker.NewServer(flGraphPath, autoRestart, enableCors, dns)
 	if err != nil {
 		return err
 	}
@@ -126,7 +127,7 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
 	for _, protoAddr := range protoAddrs {
 		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
 		if protoAddrParts[0] == "unix" {
-			syscall.Unlink(protoAddrParts[1]);
+			syscall.Unlink(protoAddrParts[1])
 		} else if protoAddrParts[0] == "tcp" {
 			if !strings.HasPrefix(protoAddrParts[1], "127.0.0.1") {
 				log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\")
@@ -147,4 +148,3 @@ func daemon(pidfile string, protoAddrs []string, autoRestart, enableCors bool, f
 	}
 	return nil
 }
-

docs/sources/api/docker_remote_api.rst

@@ -19,13 +19,38 @@ Docker Remote API
 2. Versions
 ===========
 
-The current verson of the API is 1.2
+The current verson of the API is 1.3
-Calling /images/<name>/insert is the same as calling /v1.2/images/<name>/insert
+Calling /images/<name>/insert is the same as calling /v1.3/images/<name>/insert
 You can still call an old version of the api using /v1.0/images/<name>/insert
 
+:doc:`docker_remote_api_v1.3`
+*****************************
+
+What's new
+----------
+
+Builder (/build):
+
+- Simplify the upload of the build context
+- Simply stream a tarball instead of multipart upload with 4 intermediary buffers
+- Simpler, less memory usage, less disk usage and faster
+
+.. Note::
+The /build improvements are not reverse-compatible. Pre 1.3 clients will break on /build.
+
+List containers (/containers/json):
+
+- You can use size=1 to get the size of the containers
+
+Start containers (/containers/<id>/start):
+
+- You can now pass host-specific configuration (e.g. bind mounts) in the POST body for start calls 
+
 :doc:`docker_remote_api_v1.2`
 *****************************
 
+docker v0.4.2 2e7649b_
+
 What's new
 ----------
 
@@ -65,6 +90,9 @@ Uses json stream instead of HTML hijack, it looks like this:
 	   ...
 
 
+:doc:`docker_remote_api_v1.0`
+*****************************
+
 docker v0.3.4 8d73740_
 
 What's new
@@ -75,6 +103,7 @@ Initial version
 
 .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f
 .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4
+.. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168
 
 ==================================
 Docker Remote API Client Libraries
@@ -94,6 +123,8 @@ and we will add the libraries here.
 +----------------------+----------------+--------------------------------------------+
 | Ruby                 | docker-client  | https://github.com/geku/docker-client      |
 +----------------------+----------------+--------------------------------------------+
+| Ruby                 | docker-api     | https://github.com/swipely/docker-api      |
++----------------------+----------------+--------------------------------------------+
 | Javascript           | docker-js      | https://github.com/dgoujard/docker-js      |
 +----------------------+----------------+--------------------------------------------+
 | Javascript (Angular) | dockerui       | https://github.com/crosbymichael/dockerui  |

docs/sources/api/docker_remote_api_v1.2.rst

@@ -847,7 +847,7 @@ Build an image from Dockerfile via stdin
 
 .. http:post:: /build
 
-	Build an image from Dockerfile via stdin
+	Build an image from Dockerfile
 
 	**Example request**:
 
@@ -866,9 +866,12 @@ Build an image from Dockerfile via stdin
 	   {{ STREAM }}
 
 	:query t: tag to be applied to the resulting image in case of success
+	:query remote: resource to fetch, as URI
 	:statuscode 200: no error
         :statuscode 500: server error
 
+{{ STREAM }} is the raw text output of the build command. It uses the HTTP Hijack method in order to stream.
+
 
 Check auth configuration
 ************************

docs/sources/commandline/command/build.rst

@@ -8,9 +8,11 @@
 
 ::
 
-    Usage: docker build [OPTIONS] PATH | -
+    Usage: docker build [OPTIONS] PATH | URL | -
     Build a new container image from the source code at PATH
       -t="": Tag to be applied to the resulting image in case of success.
+    When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context
+
 
 Examples
 --------
@@ -27,7 +29,15 @@ Examples
 
 .. code-block:: bash
 
-    docker build -
+    docker build - < Dockerfile
 
 | This will read a Dockerfile from Stdin without context. Due to the lack of a context, no contents of any local directory will be sent to the docker daemon.
 | ADD doesn't work when running in this mode due to the absence of the context, thus having no source files to copy to the container.
+
+
+.. code-block:: bash
+
+    docker build github.com/creack/docker-firefox
+
+| This will clone the github repository and use it as context. The Dockerfile at the root of the repository is used as Dockerfile.
+| Note that you can specify an arbitrary git repository by using the 'git://' schema.

docs/sources/commandline/command/run.rst

@@ -8,7 +8,7 @@
 
 ::
 
-    Usage: docker run [OPTIONS] IMAGE COMMAND [ARG...]
+    Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
 
     Run a command in a new container
 
@@ -25,3 +25,4 @@
       -d=[]: Set custom dns servers for the container
       -v=[]: Creates a new volume and mounts it at the specified path.
       -volumes-from="": Mount all volumes from the given container.
+      -b=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]

docs/sources/conf.py

@@ -30,6 +30,7 @@ import sys, os
 html_additional_pages = {
     'concepts/containers': 'redirect_home.html',
     'concepts/introduction': 'redirect_home.html',
+    'builder/basics': 'redirect_build.html',
     }
 
 

docs/sources/terms/fundamentals.rst

@@ -0,0 +1,97 @@
+:title: Image & Container
+:description: Definitions of an image and container
+:keywords: containers, lxc, concepts, explanation, image, container
+
+File Systems
+============
+
+.. image:: images/docker-filesystems-generic.png
+
+In order for a Linux system to run, it typically needs two `file
+systems <http://en.wikipedia.org/wiki/Filesystem>`_:
+
+1. boot file system (bootfs)
+2. root file system (rootfs)
+
+The **boot file system** contains the bootloader and the kernel. The
+user never makes any changes to the boot file system. In fact, soon
+after the boot process is complete, the entire kernel is in memory,
+and the boot file system is unmounted to free up the RAM associated
+with the initrd disk image.
+
+The **root file system** includes the typical directory structure we
+associate with Unix-like operating systems: ``/dev, /proc, /bin, /etc,
+/lib, /usr,`` and ``/tmp`` plus all the configuration files, binaries
+and libraries required to run user applications (like bash, ls, and so
+forth). 
+
+While there can be important kernal differences between different
+Linux distributions, the contents and organization of the root file
+system are usually what make your software packages dependent on one
+distribution versus another. Docker can help solve this problem by
+running multiple distributions at the same time.
+
+.. image:: images/docker-filesystems-multiroot.png
+
+Layers and Union Mounts
+=======================
+
+In a traditional Linux boot, the kernel first mounts the root file
+system as read-only, checks its integrity, and then switches the whole
+rootfs volume to read-write mode. Docker does something similar,
+*except* that instead of changing the file system to read-write mode,
+it takes advantage of a `union mount
+<http://en.wikipedia.org/wiki/Union_mount>`_ to add a read-write file
+system *over* the read-only file system. In fact there may be multiple
+read-only file systems stacked on top of each other.
+
+.. image:: images/docker-filesystems-multilayer.png
+
+At first, the top layer has nothing in it, but any time a process
+creates a file, this happens in the top layer. And if something needs
+to update an existing file in a lower layer, then the file gets copied
+to the upper layer and changes go into the copy. The version of the
+file on the lower layer cannot be seen by the applications anymore,
+but it is there, unchanged.
+
+We call the union of the read-write layer and all the read-only layers
+a **union file system**.
+
+Image
+=====
+
+In Docker terminology, a read-only layer is called an **image**. An
+image never changes. Because Docker uses a union file system, the
+applications think the whole file system is mounted read-write,
+because any file can be changed. But all the changes go to the
+top-most layer, and underneath, the image is unchanged. Since they
+don't change, images do not have state.
+
+Each image may depend on one more image which forms the layer beneath
+it. We sometimes say that the lower image is the **parent** of the
+upper image.
+
+Base Image
+==========
+
+An image that has no parent is a **base image**.
+
+Container
+=========
+
+Once you start a process in Docker from an image, Docker fetches the
+image and its parent, and repeats the process until it reaches the
+base image. Then the union file system adds a read-write layer on
+top. That read-write layer, plus the information about its parent and
+some additional information like its unique id, is called a
+**container**. 
+
+Containers can change, and so they have state. A container may be
+running or exited. In either case, the state of the file system and
+its exit value is preserved. You can start, stop, and restart a
+container. The processes restart from scratch (their memory state is
+**not** preserved in a container), but the file system is just as it
+was when the container was stopped.
+
+You can promote a container to an image with ``docker commit``. Once a
+container is an image, you can use it as a parent for new containers.

docs/sources/terms/index.rst

@@ -0,0 +1,18 @@
+:title: Terms
+:description: Definitions of terms used in Docker documentation
+:keywords: concepts, documentation, docker, containers
+
+
+
+Terms
+=====
+
+Definitions of terms used in Docker documentation.
+
+Contents:
+
+.. toctree::
+   :maxdepth: 1
+
+   fundamentals
+

docs/sources/toctree.rst

@@ -18,5 +18,6 @@ This documentation has the following resources:
    contributing/index
    api/index
    faq
+   terms/index
 
 .. image:: concepts/images/lego_docker.jpg

docs/sources/use/builder.rst

@@ -121,19 +121,7 @@ functionally equivalent to prefixing the command with `<key>=<value>`
 .. note::
     The environment variables will persist when a container is run from the resulting image.
 
-2.7 INSERT
+2.7 ADD
-----------
-
-    ``INSERT <file url> <path>``
-
-The `INSERT` instruction will download the file from the given url to the given
-path within the image. It is similar to `RUN curl -o <path> <url>`, assuming 
-curl was installed within the image.
-
-.. note::
-    The path must include the file name.
-
-2.8 ADD
 -------
 
     ``ADD <src> <dest>``
@@ -141,7 +129,7 @@ curl was installed within the image.
 The `ADD` instruction will copy new files from <src> and add them to the container's filesystem at path `<dest>`.
 
 `<src>` must be the path to a file or directory relative to the source directory being built (also called the
-context of the build).
+context of the build) or a remote file URL.
 
 `<dest>` is the path at which the source will be copied in the destination container.
 
@@ -182,7 +170,6 @@ files and directories are created with mode 0700, uid and gid 0.
     RUN apt-get update
     
     RUN apt-get install -y inotify-tools nginx apache2 openssh-server
-    INSERT https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper
 
 .. code-block:: bash
 

docs/theme/docker/layout.html

@@ -40,8 +40,11 @@
 
     {%- set script_files = script_files + ['_static/js/docs.js'] %}
 
+    {%- if pagename == 'index' %}
+    <link rel="canonical" href="http://docs.docker.io/en/latest/">
+    {% else %}
     <link rel="canonical" href="http://docs.docker.io/en/latest/{{ pagename }}/">
-
+    {% endif %}
     {%- for cssfile in css_files %}
     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
     {%- endfor %}

docs/theme/docker/redirect_build.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Page Moved</title>
+    <meta http-equiv="refresh" content="0; url=http://docs.docker.io/en/latest/use/builder/">
+</head>
+<body>
+
+This page has moved. Perhaps you should visit the <a href="http://docs.docker.io/en/latest/use/builder/" title="builder page">Builder page</a>
+
+</body>
+</html>

docs/theme/docker/redirect_home.html

@@ -2,7 +2,7 @@
 <html>
 <head>
     <title>Page Moved</title>
-    <meta http-equiv="refresh" content="0; url=http://docks.docker.io/en/latest/">
+    <meta http-equiv="refresh" content="0; url=http://docs.docker.io/en/latest/">
 </head>
 <body>
 

graph.go

@@ -189,7 +189,7 @@ func (graph *Graph) Mktemp(id string) (string, error) {
 		return "", fmt.Errorf("Couldn't create temp: %s", err)
 	}
 	if tmp.Exists(id) {
-		return "", fmt.Errorf("Image %d already exists", id)
+		return "", fmt.Errorf("Image %s already exists", id)
 	}
 	return tmp.imageRoot(id), nil
 }

hack/README.rst

@@ -3,8 +3,8 @@ This directory contains material helpful for hacking on docker.
 make hack
 =========
 
-Set up an Ubuntu 13.04 virtual machine for developers including kernel 3.8
+Set up an Ubuntu 12.04 virtual machine for developers including kernel 3.8
-and buildbot. The environment is setup in a way that can be used through
+go1.1 and buildbot. The environment is setup in a way that can be used through
 the usual go workflow and/or the root Makefile. You can either edit on
 your host, or inside the VM (using make ssh-dev) and run and test docker
 inside the VM.
@@ -22,6 +22,7 @@ developers are inconvenienced by the failure.
 
 When running 'make hack' at the docker root directory, it spawns a virtual
 machine in the background running a buildbot instance and adds a git
-post-commit hook that automatically run docker tests for you.
+post-commit hook that automatically run docker tests for you each time you
+commit in your local docker repository.
 
 You can check your buildbot instance at http://192.168.33.21:8010/waterfall

hack/RELEASE.md

@@ -0,0 +1,119 @@
+## A maintainer's guide to releasing Docker
+
+So you're in charge of a docker release? Cool. Here's what to do.
+
+If your experience deviates from this document, please document the changes to keep it
+up-to-date.
+
+
+### 1. Pull from master and create a release branch
+
+	```bash
+	$ git checkout master
+	$ git pull
+	$ git checkout -b bump_$VERSION
+	```
+
+### 2. Update CHANGELOG.md
+
+	You can run this command for reference:
+
+	```bash
+	LAST_VERSION=$(git tag | grep -E "v[0-9\.]+$" | sort -nr | head -n 1)
+	git log $LAST_VERSION..HEAD
+	```
+
+	Each change should be formatted as ```BULLET CATEGORY: DESCRIPTION```
+
+	* BULLET is either ```-```, ```+``` or ```*```, to indicate a bugfix,
+	new feature or upgrade, respectively.
+
+	* CATEGORY should describe which part of the project is affected.
+	Valid categories are:
+		* Runtime
+		* Remote API
+		* Builder
+		* Documentation
+		* Hack
+
+	* DESCRIPTION: a concise description of the change that is relevant to the end-user,
+	using the present tense.
+	Changes should be described in terms of how they affect the user, for example "new feature
+	X which allows Y", "fixed bug which caused X", "increased performance of Y".
+
+	EXAMPLES:
+
+		```
+		 + Builder: 'docker build -t FOO' applies the tag FOO to the newly built container.
+		 * Runtime: improve detection of kernel version
+		 - Remote API: fix a bug in the optional unix socket transport
+		 ```
+
+### 3. Change VERSION in commands.go
+
+### 4. Run all tests
+
+### 5. Commit and create a pull request
+
+	```bash
+	$ git add commands.go CHANGELOG.md
+	$ git commit -m "Bump version to $VERSION"
+	$ git push origin bump_$VERSION
+	```
+
+### 6. Get 2 other maintainers to validate the pull request
+
+### 7. Merge the pull request and apply tags
+
+	```bash
+	$ git checkout master
+	$ git merge bump_$VERSION
+	$ git tag -a v$VERSION # Don't forget the v!
+	$ git tag -f -a latest
+	$ git push
+	$ git push --tags
+	```
+
+### 8. Publish binaries
+
+	To run this you will need access to the release credentials.
+	Get them from [the infrastructure maintainers](https://github.com/dotcloud/docker/blob/master/hack/infrastructure/MAINTAINERS).
+
+	```bash
+	$ RELEASE_IMAGE=image_provided_by_infrastructure_maintainers
+	$ BUILD=$(docker run -d -e RELEASE_PPA=0 $RELEASE_IMAGE)
+	```
+
+	This will do 2 things:
+	
+	* It will build and upload the binaries on http://get.docker.io
+	* It will *test* the release on our Ubuntu PPA (a PPA is a community repository for ubuntu packages)
+
+	Wait for the build to complete.
+
+	```bash
+	$ docker wait $BUILD # This should print 0. If it doesn't, your build failed.
+	```
+
+	Check that the output looks OK. Here's an example of a correct output:
+
+	```bash
+	$ docker logs 2>&1 b4e7c8299d73 | grep -e 'Public URL' -e 'Successfully uploaded'
+	Public URL of the object is: http://get.docker.io.s3.amazonaws.com/builds/Linux/x86_64/docker-v0.4.7.tgz
+	Public URL of the object is: http://get.docker.io.s3.amazonaws.com/builds/Linux/x86_64/docker-latest.tgz
+	Successfully uploaded packages.
+	```
+
+	If you don't see 3 lines similar to this, something might be wrong. Check the full logs and try again.
+	
+
+### 9. Publish Ubuntu packages
+
+	If everything went well in the previous step, you can finalize the release by submitting the Ubuntu packages.
+
+	```bash
+	$ RELEASE_IMAGE=image_provided_by_infrastructure_maintainers
+	$ docker run -e RELEASE_PPA=1 $RELEASE_IMAGE
+	```
+
+	If that goes well, congratulations! You're done.

hack/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 BOX_NAME = "ubuntu-dev"
-BOX_URI = "http://cloud-images.ubuntu.com/raring/current/raring-server-cloudimg-vagrant-amd64-disk1.box"
+BOX_URI = "http://files.vagrantup.com/precise64.box"
 VM_IP = "192.168.33.21"
 USER = "vagrant"
 GOPATH = "/data/docker"
@@ -21,14 +21,15 @@ Vagrant::Config.run do |config|
   # Touch for makefile
   pkg_cmd = "touch #{DOCKER_PATH}; "
   # Install docker dependencies
-  pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \
+  pkg_cmd << "apt-get update -qq; apt-get install -y python-software-properties; " \
-    "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
+    "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \
+    "apt-get install -y linux-image-generic-lts-raring lxc git aufs-tools golang-stable make; " \
     "chown -R #{USER}.#{USER} #{GOPATH}; " \
     "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile"
   config.vm.provision :shell, :inline => pkg_cmd
   # Deploy buildbot CI
   pkg_cmd = "apt-get install -q -y python-dev python-pip supervisor; " \
-    "pip install -r #{CFG_PATH}/requirements.txt; " \
+    "pip install -q -r #{CFG_PATH}/requirements.txt; " \
     "chown #{USER}.#{USER} /data; cd /data; " \
     "#{CFG_PATH}/setup.sh #{USER} #{GOPATH} #{DOCKER_PATH} #{CFG_PATH} #{BUILDBOT_PATH}"
   config.vm.provision :shell, :inline => pkg_cmd

image.go

@@ -92,10 +92,12 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error {
 		defer file.Close()
 		layerData = file
 	}
-
+	// If layerData is not nil, unpack it into the new layer
+	if layerData != nil {
 		if err := Untar(layerData, layer); err != nil {
 			return err
 		}
+	}
 
 	return StoreSize(img, root)
 }

lxc_template.go

@@ -84,8 +84,9 @@ lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0
 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container
 lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0
 {{if .Volumes}}
-{{range $virtualPath, $realPath := .GetVolumes}}
+{{ $rw := .VolumesRW }}
-lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,rw 0 0
+{{range $virtualPath, $realPath := .Volumes}}
+lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,{{ if index $rw $virtualPath }}rw{{else}}ro{{end}} 0 0
 {{end}}
 {{end}}
 

network.go

@@ -257,7 +257,6 @@ func proxy(listener net.Listener, proto, address string) error {
 		utils.Debugf("Connected to backend, splicing")
 		splice(src, dst)
 	}
-	panic("Unreachable")
 }
 
 func halfSplice(dst, src net.Conn) error {

packaging/ubuntu/Makefile

@@ -23,7 +23,7 @@ install:
 	mkdir -p ${DESTDIR}/etc/init
 	mkdir -p ${DESTDIR}/DEBIAN
 	install -m 0755 src/${GITHUB_PATH}/docker/docker ${DESTDIR}/usr/bin
-	install -o root -m 0755 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf
+	install -o root -m 0644 debian/docker.upstart ${DESTDIR}/etc/init/docker.conf
 	install debian/lxc-docker.prerm ${DESTDIR}/DEBIAN/prerm
 	install debian/lxc-docker.postinst ${DESTDIR}/DEBIAN/postinst
 

registry/registry.go

@@ -18,6 +18,14 @@ import (
 
 var ErrAlreadyExists = errors.New("Image already exists")
 
+func UrlScheme() string {
+	u, err := url.Parse(auth.IndexServerAddress())
+	if err != nil {
+		return "https"
+	}
+	return u.Scheme
+}
+
 func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
 	for _, cookie := range c.Jar.Cookies(req.URL) {
 		req.AddCookie(cookie)
@@ -56,20 +64,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s
 }
 
 // Check if an image exists in the Registry
-func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool {
+func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool {
 	rt := &http.Transport{Proxy: http.ProxyFromEnvironment}
 
 	req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil)
 	if err != nil {
 		return false
 	}
-	req.SetBasicAuth(authConfig.Username, authConfig.Password)
 	res, err := rt.RoundTrip(req)
 	if err != nil {
 		return false
 	}
 	res.Body.Close()
-	return res.StatusCode == 307
+	return res.StatusCode == 200
 }
 
 func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) {
@@ -155,7 +162,10 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
 		repository = "library/" + repository
 	}
 	for _, host := range registries {
-		endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository)
+		endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository)
+		if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) {
+			endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint)
+		}
 		req, err := r.opaqueRequest("GET", endpoint, nil)
 		if err != nil {
 			return nil, err
@@ -165,6 +175,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
 		if err != nil {
 			return nil, err
 		}
+
 		utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint)
 		defer res.Body.Close()
 
@@ -249,7 +260,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
 
 // Push a local image to the registry
 func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 	// FIXME: try json with UTF8
 	req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw)))
 	if err != nil {
@@ -285,7 +296,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis
 }
 
 func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error {
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 	req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer)
 	if err != nil {
 		return err
@@ -323,7 +334,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R
 func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error {
 	// "jsonify" the string
 	revision = "\"" + revision + "\""
-	registry = "https://" + registry + "/v1"
+	registry = registry + "/v1"
 
 	req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
 	if err != nil {

runtime.go

@@ -144,7 +144,9 @@ func (runtime *Runtime) Register(container *Container) error {
 				utils.Debugf("Restarting")
 				container.State.Ghost = false
 				container.State.setStopped(0)
-				if err := container.Start(); err != nil {
+				// assume empty host config
+				hostConfig := &HostConfig{}
+				if err := container.Start(hostConfig); err != nil {
 					return err
 				}
 				nomonitor = true
@@ -246,8 +248,8 @@ func (runtime *Runtime) UpdateCapabilities(quiet bool) {
 }
 
 // FIXME: harmonize with NewGraph()
-func NewRuntime(autoRestart bool, dns []string) (*Runtime, error) {
+func NewRuntime(flGraphPath string, autoRestart bool, dns []string) (*Runtime, error) {
-	runtime, err := NewRuntimeFromDirectory("/var/lib/docker", autoRestart)
+	runtime, err := NewRuntimeFromDirectory(flGraphPath, autoRestart)
 	if err != nil {
 		return nil, err
 	}

runtime_test.go

@@ -8,17 +8,23 @@ import (
 	"log"
 	"net"
 	"os"
-	"os/user"
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 )
 
-const unitTestImageName string = "docker-ut"
+const (
-const unitTestImageId string = "e9aa60c60128cad1"
+	unitTestImageName = "docker-unit-tests"
-const unitTestStoreBase string = "/var/lib/docker/unit-tests"
+	unitTestImageId   = "e9aa60c60128cad1"
+	unitTestStoreBase = "/var/lib/docker/unit-tests"
+	testDaemonAddr    = "127.0.0.1:4270"
+	testDaemonProto   = "tcp"
+)
+
+var globalRuntime *Runtime
 
 func nuke(runtime *Runtime) error {
 	var wg sync.WaitGroup
@@ -33,6 +39,23 @@ func nuke(runtime *Runtime) error {
 	return os.RemoveAll(runtime.root)
 }
 
+func cleanup(runtime *Runtime) error {
+	for _, container := range runtime.List() {
+		container.Kill()
+		runtime.Destroy(container)
+	}
+	images, err := runtime.graph.All()
+	if err != nil {
+		return err
+	}
+	for _, image := range images {
+		if image.ID != unitTestImageId {
+			runtime.graph.Delete(image.ID)
+		}
+	}
+	return nil
+}
+
 func layerArchive(tarfile string) (io.Reader, error) {
 	// FIXME: need to close f somewhere
 	f, err := os.Open(tarfile)
@@ -49,10 +72,8 @@ func init() {
 		return
 	}
 
-	if usr, err := user.Current(); err != nil {
+	if uid := syscall.Geteuid(); uid != 0 {
-		panic(err)
+		log.Fatal("docker tests needs to be run as root")
-	} else if usr.Uid != "0" {
-		panic("docker tests needs to be run as root")
 	}
 
 	NetworkBridgeIface = "testdockbr0"
@@ -62,6 +83,7 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+	globalRuntime = runtime
 
 	// Create the "Server"
 	srv := &Server{
@@ -75,6 +97,16 @@ func init() {
 	if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
 		panic(err)
 	}
+
+	// Spawn a Daemon
+	go func() {
+		if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil {
+			panic(err)
+		}
+	}()
+
+	// Give some time to ListenAndServer to actually start
+	time.Sleep(time.Second)
 }
 
 // FIXME: test that ImagePull(json=true) send correct json output
@@ -103,10 +135,13 @@ func GetTestImage(runtime *Runtime) *Image {
 	imgs, err := runtime.graph.All()
 	if err != nil {
 		panic(err)
-	} else if len(imgs) < 1 {
-		panic("GASP")
 	}
-	return imgs[0]
+	for i := range imgs {
+		if imgs[i].ID == unitTestImageId {
+			return imgs[i]
+		}
+	}
+	panic(fmt.Errorf("Test image %v not found", unitTestImageId))
 }
 
 func TestRuntimeCreate(t *testing.T) {
@@ -295,7 +330,8 @@ func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err := container.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container.Start(hostConfig); err != nil {
 		if strings.Contains(err.Error(), "address already in use") {
 			return nil, nil
 		}
@@ -405,7 +441,8 @@ func TestRestore(t *testing.T) {
 	defer runtime1.Destroy(container2)
 
 	// Start the container non blocking
-	if err := container2.Start(); err != nil {
+	hostConfig := &HostConfig{}
+	if err := container2.Start(hostConfig); err != nil {
 		t.Fatal(err)
 	}
 

server.go

@@ -87,7 +87,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.
 	}
 	defer file.Body.Close()
 
-	config, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities)
+	config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities)
 	if err != nil {
 		return "", err
 	}
@@ -254,7 +254,7 @@ func (srv *Server) ContainerChanges(name string) ([]Change, error) {
 	return nil, fmt.Errorf("No such container: %s", name)
 }
 
-func (srv *Server) Containers(all bool, n int, since, before string) []APIContainers {
+func (srv *Server) Containers(all, size bool, n int, since, before string) []APIContainers {
 	var foundBefore bool
 	var displayed int
 	retContainers := []APIContainers{}
@@ -288,8 +288,9 @@ func (srv *Server) Containers(all bool, n int, since, before string) []APIContai
 		c.Created = container.Created.Unix()
 		c.Status = container.State.String()
 		c.Ports = container.NetworkSettings.PortMappingHuman()
+		if size {
 			c.SizeRw, c.SizeRootFs = container.GetSize()
-
+		}
 		retContainers = append(retContainers, c)
 	}
 	return retContainers
@@ -350,9 +351,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin
 	return nil
 }
 
-func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag string, sf *utils.StreamFormatter) error {
+func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error {
 	out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress()))
-	repoData, err := r.GetRepositoryData(remote)
+
+	var repoData *registry.RepositoryData
+	var err error
+	if registryEp == "" {
+		repoData, err = r.GetRepositoryData(remote)
 		if err != nil {
 			return err
 		}
@@ -362,14 +367,33 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
 		if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil {
 			return err
 		}
+	} else {
+		repoData = &registry.RepositoryData{
+			Tokens:    []string{},
+			ImgList:   make(map[string]*registry.ImgData),
+			Endpoints: []string{registryEp},
+		}
+	}
 
 	utils.Debugf("Retrieving the tag list")
 	tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
 	if err != nil {
+		utils.Debugf("%v", err)
 		return err
 	}
+
+	if registryEp != "" {
+		for tag, id := range tagsList {
+			repoData.ImgList[id] = &registry.ImgData{
+				ID:       id,
+				Tag:      tag,
+				Checksum: "",
+			}
+		}
+	}
+
 	utils.Debugf("Registering tags")
-	// If not specific tag have been asked, take all
+	// If no tag has been specified, pull them all
 	if askedTag == "" {
 		for tag, id := range tagsList {
 			repoData.ImgList[id].Tag = tag
@@ -391,7 +415,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re
 		out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote))
 		success := false
 		for _, ep := range repoData.Endpoints {
-			if err := srv.pullImage(r, out, img.ID, "https://"+ep+"/v1", repoData.Tokens, sf); err != nil {
+			if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
+				ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
+			}
+			if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil {
 				out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
 				continue
 			}
@@ -451,7 +478,6 @@ func (srv *Server) poolRemove(kind, key string) error {
 	}
 	return nil
 }
-
 func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
 	r, err := registry.NewRegistry(srv.runtime.root, authConfig)
 	if err != nil {
@@ -462,24 +488,23 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util
 	}
 	defer srv.poolRemove("pull", name+":"+tag)
 
-	out = utils.NewWriteFlusher(out)
-	if endpoint != "" {
-		if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil {
-			return err
-		}
-		return nil
-	}
 	remote := name
 	parts := strings.Split(name, "/")
 	if len(parts) > 2 {
 		remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
 	}
-	if err := srv.pullRepository(r, out, name, remote, tag, sf); err != nil {
+	out = utils.NewWriteFlusher(out)
+	err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf)
+	if err != nil && endpoint != "" {
+		if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil {
 			return err
 		}
 		return nil
 	}
 
+	return nil
+}
+
 // Retrieve the checksum of an image
 // Priority:
 // - Check on the stored checksums
@@ -546,7 +571,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat
 	return imgList, nil
 }
 
-func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error {
+func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error {
 	out = utils.NewWriteFlusher(out)
 	out.Write(sf.FormatStatus("Processing checksums"))
 	imgList, err := srv.getImageList(localRepo)
@@ -554,25 +579,51 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri
 		return err
 	}
 	out.Write(sf.FormatStatus("Sending image list"))
-
 	srvName := name
 	parts := strings.Split(name, "/")
 	if len(parts) > 2 {
 		srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/")))
 	}
 
-	repoData, err := r.PushImageJSONIndex(srvName, imgList, false, nil)
+	var repoData *registry.RepositoryData
+	if registryEp == "" {
+		repoData, err = r.PushImageJSONIndex(name, imgList, false, nil)
 		if err != nil {
 			return err
 		}
+	} else {
+		repoData = &registry.RepositoryData{
+			ImgList:   make(map[string]*registry.ImgData),
+			Tokens:    []string{},
+			Endpoints: []string{registryEp},
+		}
+		tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens)
+		if err != nil && err.Error() != "Repository not found" {
+			return err
+		} else if err == nil {
+			for tag, id := range tagsList {
+				repoData.ImgList[id] = &registry.ImgData{
+					ID:       id,
+					Tag:      tag,
+					Checksum: "",
+				}
+			}
+		}
+	}
 
 	for _, ep := range repoData.Endpoints {
+		if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) {
+			ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep)
+		}
 		out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo)))
 		// For each image within the repo, push them
 		for _, elem := range imgList {
 			if _, exists := repoData.ImgList[elem.ID]; exists {
 				out.Write(sf.FormatStatus("Image %s already on registry, skipping", name))
 				continue
+			} else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) {
+				fmt.Fprintf(out, "Image %s already on registry, skipping\n", name)
+				continue
 			}
 			if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil {
 				// FIXME: Continue on error?
@@ -585,9 +636,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri
 		}
 	}
 
-	if _, err := r.PushImageJSONIndex(srvName, imgList, true, repoData.Endpoints); err != nil {
+	if registryEp == "" {
+		if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil {
 			return err
 		}
+	}
+
 	return nil
 }
 
@@ -664,11 +718,12 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str
 	if err2 != nil {
 		return err2
 	}
+
 	if err != nil {
 		out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name])))
 		// If it fails, try to get the repository
 		if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists {
-			if err := srv.pushRepository(r, out, name, localRepo, sf); err != nil {
+			if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil {
 				return err
 			}
 			return nil
@@ -857,7 +912,7 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error {
 	return nil
 }
 
-func (srv *Server) deleteImage(img *Image, repoName, tag string) (*[]APIRmi, error) {
+func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, error) {
 	//Untag the current image
 	var imgs []APIRmi
 	tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag)
@@ -870,18 +925,18 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) (*[]APIRmi, err
 	if len(srv.runtime.repositories.ByID()[img.ID]) == 0 {
 		if err := srv.deleteImageAndChildren(img.ID, &imgs); err != nil {
 			if err != ErrImageReferenced {
-				return &imgs, err
+				return imgs, err
 			}
 		} else if err := srv.deleteImageParents(img, &imgs); err != nil {
 			if err != ErrImageReferenced {
-				return &imgs, err
+				return imgs, err
 			}
 		}
 	}
-	return &imgs, nil
+	return imgs, nil
 }
 
-func (srv *Server) ImageDelete(name string, autoPrune bool) (*[]APIRmi, error) {
+func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) {
 	img, err := srv.runtime.repositories.LookupImage(name)
 	if err != nil {
 		return nil, fmt.Errorf("No such image: %s", name)
@@ -933,9 +988,9 @@ func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error)
 	return nil, nil
 }
 
-func (srv *Server) ContainerStart(name string) error {
+func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error {
 	if container := srv.runtime.Get(name); container != nil {
-		if err := container.Start(); err != nil {
+		if err := container.Start(hostConfig); err != nil {
 			return fmt.Errorf("Error starting container %s: %s", name, err.Error())
 		}
 	} else {
@@ -1048,11 +1103,11 @@ func (srv *Server) ImageInspect(name string) (*Image, error) {
 	return nil, fmt.Errorf("No such image: %s", name)
 }
 
-func NewServer(autoRestart, enableCors bool, dns ListOpts) (*Server, error) {
+func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (*Server, error) {
 	if runtime.GOARCH != "amd64" {
 		log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH)
 	}
-	runtime, err := NewRuntime(autoRestart, dns)
+	runtime, err := NewRuntime(flGraphPath, autoRestart, dns)
 	if err != nil {
 		return nil, err
 	}

server_test.go

@@ -65,7 +65,7 @@ func TestCreateRm(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	config, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -98,7 +98,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	config, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil)
+	config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +112,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
 	}
 
-	err = srv.ContainerStart(id)
+	err = srv.ContainerStart(id, hostConfig)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -127,7 +127,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = srv.ContainerStart(id)
+	err = srv.ContainerStart(id, hostConfig)
 	if err != nil {
 		t.Fatal(err)
 	}

term/term.go

@@ -38,13 +38,13 @@ func IsTerminal(fd uintptr) bool {
 
 // Restore restores the terminal connected to the given file descriptor to a
 // previous state.
-func Restore(fd uintptr, state *State) error {
+func RestoreTerminal(fd uintptr, state *State) error {
 	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)))
 	return err
 }
 
-func SetRawTerminal() (*State, error) {
+func SetRawTerminal(fd uintptr) (*State, error) {
-	oldState, err := MakeRaw(os.Stdin.Fd())
+	oldState, err := MakeRaw(fd)
 	if err != nil {
 		return nil, err
 	}
@@ -52,12 +52,8 @@ func SetRawTerminal() (*State, error) {
 	signal.Notify(c, os.Interrupt)
 	go func() {
 		_ = <-c
-		Restore(os.Stdin.Fd(), oldState)
+		RestoreTerminal(fd, oldState)
 		os.Exit(0)
 	}()
 	return oldState, err
 }
-
-func RestoreTerminal(state *State) {
-	Restore(os.Stdin.Fd(), state)
-}

testing/Vagrantfile

@@ -19,9 +19,10 @@ Vagrant::Config.run do |config|
   config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
   config.vm.network :hostonly, BUILDBOT_IP
 
+
   # Deploy buildbot and its dependencies if it was not done
   if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
-    pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; "
+    pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; "
     # Deploy buildbot CI
     pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
       "pip install -r #{CFG_PATH}/requirements.txt; " \
@@ -29,7 +30,7 @@ Vagrant::Config.run do |config|
       "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; "
     # Install docker dependencies
     pkg_cmd << "apt-get install -q -y python-software-properties; " \
-      "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \
+      "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \
       "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; "
     # Activate new kernel
     pkg_cmd << "shutdown -r +1; "
@@ -40,6 +41,7 @@ end
 # Providers were added on Vagrant >= 1.1.0
 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
   config.vm.provider :aws do |aws, override|
+    aws.tags = { 'Name' => 'docker-ci' }
     aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
     aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
     aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]

utils/utils.go

@@ -78,16 +78,16 @@ func (r *progressReader) Read(p []byte) (n int, err error) {
 	read, err := io.ReadCloser(r.reader).Read(p)
 	r.readProgress += read
 
-	updateEvery := 4096
+	updateEvery := 1024 * 512 //512kB
 	if r.readTotal > 0 {
-		// Only update progress for every 1% read
+		// Update progress for every 1% read if 1% < 512kB
-		if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
+		if increment := int(0.01 * float64(r.readTotal)); increment < updateEvery {
 			updateEvery = increment
 		}
 	}
 	if r.readProgress-r.lastUpdate > updateEvery || err != nil {
 		if r.readTotal > 0 {
-			fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
+			fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
 		} else {
 			fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
 		}
@@ -133,7 +133,7 @@ func HumanDuration(d time.Duration) string {
 	} else if hours < 24*365*2 {
 		return fmt.Sprintf("%d months", hours/24/30)
 	}
-	return fmt.Sprintf("%d years", d.Hours()/24/365)
+	return fmt.Sprintf("%f years", d.Hours()/24/365)
 }
 
 // HumanSize returns a human-readable approximation of a size
@@ -147,7 +147,7 @@ func HumanSize(size int64) string {
 		sizef = sizef / 1000.0
 		i++
 	}
-	return fmt.Sprintf("%.4g %s", sizef, units[i])
+	return fmt.Sprintf("%5.4g %s", sizef, units[i])
 }
 
 func Trunc(s string, maxlen int) string {
@@ -236,7 +236,6 @@ func (r *bufReader) Read(p []byte) (n int, err error) {
 		}
 		r.wait.Wait()
 	}
-	panic("unreachable")
 }
 
 func (r *bufReader) Close() error {
@@ -529,7 +528,9 @@ func GetKernelVersion() (*KernelVersionInfo, error) {
 	}
 
 	if len(tmp2) > 2 {
-		minor, err = strconv.Atoi(tmp2[2])
+		// Removes "+" because git kernels might set it
+		minorUnparsed := strings.Trim(tmp2[2], "+")
+		minor, err = strconv.Atoi(minorUnparsed)
 		if err != nil {
 			return nil, err
 		}
@@ -637,6 +638,14 @@ func (sf *StreamFormatter) Used() bool {
 	return sf.used
 }
 
+func IsURL(str string) bool {
+	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://")
+}
+
+func IsGIT(str string) bool {
+	return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/")
+}
+
 func CheckLocalDns() bool {
 	resolv, err := ioutil.ReadFile("/etc/resolv.conf")
 	if err != nil {
@@ -678,5 +687,3 @@ func ParseHost(host string, port int, addr string) string {
 	}
 	return fmt.Sprintf("tcp://%s:%d", host, port)
 }
-
-

utils_test.go

@@ -0,0 +1,103 @@
+package docker
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"testing"
+)
+
+// This file contains utility functions for docker's unit test suite.
+// It has to be named XXX_test.go, apparently, in other to access private functions
+// from other XXX_test.go functions.
+
+// Create a temporary runtime suitable for unit testing.
+// Call t.Fatal() at the first error.
+func mkRuntime(t *testing.T) *Runtime {
+	runtime, err := newTestRuntime()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return runtime
+}
+
+// Write `content` to the file at path `dst`, creating it if necessary,
+// as well as any missing directories.
+// The file is truncated if it already exists.
+// Call t.Fatal() at the first error.
+func writeFile(dst, content string, t *testing.T) {
+	// Create subdirectories if necessary
+	if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) {
+		t.Fatal(err)
+	}
+	f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Write content (truncate if it exists)
+	if _, err := io.Copy(f, strings.NewReader(content)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Return the contents of file at path `src`.
+// Call t.Fatal() at the first error (including if the file doesn't exist)
+func readFile(src string, t *testing.T) (content string) {
+	f, err := os.Open(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(data)
+}
+
+// Create a test container from the given runtime `r` and run arguments `args`.
+// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image.
+// The caller is responsible for destroying the container.
+// Call t.Fatal() at the first error.
+func mkContainer(r *Runtime, args []string, t *testing.T) (*Container, *HostConfig) {
+	config, hostConfig, _, err := ParseRun(args, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.Image = GetTestImage(r).ID
+	c, err := NewBuilder(r).Create(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c, hostConfig
+}
+
+// Create a test container, start it, wait for it to complete, destroy it,
+// and return its standard output as a string.
+// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image.
+// If t is not nil, call t.Fatal() at the first error. Otherwise return errors normally.
+func runContainer(r *Runtime, args []string, t *testing.T) (output string, err error) {
+	defer func() {
+		if err != nil && t != nil {
+			t.Fatal(err)
+		}
+	}()
+	container, hostConfig := mkContainer(r, args, t)
+	defer r.Destroy(container)
+	stdout, err := container.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	defer stdout.Close()
+	if err := container.Start(hostConfig); err != nil {
+		return "", err
+	}
+	container.Wait()
+	data, err := ioutil.ReadAll(stdout)
+	if err != nil {
+		return "", err
+	}
+	output = string(data)
+	return
+}

z_final_test.go

@@ -0,0 +1,12 @@
+package docker
+
+import (
+	"github.com/dotcloud/docker/utils"
+	"runtime"
+	"testing"
+)
+
+func TestFinal(t *testing.T) {
+	cleanup(globalRuntime)
+	t.Logf("Fds: %d, Goroutines: %d", utils.GetTotalUsedFds(), runtime.NumGoroutine())
+}