From 26533eb2c419155b93e3f403d081fb3c52d4dc45 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Thu, 14 Nov 2013 05:34:25 +0000 Subject: [PATCH 001/162] Adding a makefile --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..a207424e77 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +default: build + +build: + sudo docker build -t docker . + sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh binary + +doc: + cd docs && docker build -t docker:docs . && docker run -p 8000:8000 docker:docs + +test: + sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test + +shell: + sudo docker run -privileged -i -t docker bash From 7267c4b746ad3b86d1174aeed4c06679476e8a27 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Thu, 14 Nov 2013 05:53:53 +0000 Subject: [PATCH 002/162] Removing sudo --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a207424e77..1447a4ed12 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ default: build build: - sudo docker build -t docker . - sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh binary + docker build -t docker . + docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh binary doc: cd docs && docker build -t docker:docs . && docker run -p 8000:8000 docker:docs test: - sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test + docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test shell: - sudo docker run -privileged -i -t docker bash + docker run -privileged -i -t docker bash From b3bee7e0c43dd281c081cd59d7403148d3f8af88 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Wed, 20 Nov 2013 18:45:12 +0000 Subject: [PATCH 003/162] utils: remove dotcloud/tar dep --- .../infrastructure/docker-ci/docker-coverage/coverage-docker.sh | 2 +- hack/vendor.sh | 2 -- utils/tarsum.go | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh b/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh index 2ca3b6e801..f03243cf8f 100755 --- a/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh +++ b/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh @@ -23,7 +23,7 @@ cd $BASE_PATH/go GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\ -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\ -code.google.com/p/go.net/websocket,github.com/dotcloud/tar\ +code.google.com/p/go.net/websocket\ github.com/dotcloud/docker | ./bin/gocov report; exit_status=$? # Cleanup testing directory diff --git a/hack/vendor.sh b/hack/vendor.sh index a2411c8005..93dd8739ce 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -27,8 +27,6 @@ git_clone github.com/gorilla/context/ 708054d61e5 git_clone github.com/gorilla/mux/ 9b36453141c -git_clone github.com/dotcloud/tar/ e5ea6bb21a - # Docker requires code.google.com/p/go.net/websocket PKG=code.google.com/p/go.net REV=84a4013f96e0 ( diff --git a/utils/tarsum.go b/utils/tarsum.go index 290be241a9..9cba516a34 100644 --- a/utils/tarsum.go +++ b/utils/tarsum.go @@ -5,7 +5,7 @@ import ( "compress/gzip" "crypto/sha256" "encoding/hex" - "github.com/dotcloud/tar" + "archive/tar" "hash" "io" "sort" From f1e6dce047d091bc4b4ea9264a29401f39381fa6 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 21 Nov 2013 16:19:19 -0700 Subject: [PATCH 004/162] Update test scripts to always run ALL tests, even when some fail --- hack/make/dyntest | 39 ++++++++++++++++++++++++++++----------- hack/make/test | 37 +++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/hack/make/dyntest b/hack/make/dyntest index ff607d3910..756b038207 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -19,18 +19,35 @@ fi bundle_test() { { date - for test_dir in $(find_test_dirs); do ( - set -x - cd $test_dir + + TESTS_FAILED=() + for test_dir in $(find_test_dirs); do + echo - # Install packages that are dependencies of the tests. - # Note: Does not run the tests. - go test -i -ldflags "$LDFLAGS" $BUILDFLAGS - - # Run the tests with the optional $TESTFLAGS. - export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION - go test -v -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS - ) done + if ! ( + set -x + cd $test_dir + + # Install packages that are dependencies of the tests. + # Note: Does not run the tests. + go test -i -ldflags "$LDFLAGS" $BUILDFLAGS + + # Run the tests with the optional $TESTFLAGS. + export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION + go test -v -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS + ); then + TESTS_FAILED+=("$test_dir") + sleep 1 # give it a second, so observers watching can take note + fi + done + + # if some tests fail, we want the bundlescript to fail, but we want to + # try running ALL the tests first, hence TESTS_FAILED + if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then + echo + echo "Test failures in: ${TESTS_FAILED[@]}" + false + fi } 2>&1 | tee $DEST/test.log } diff --git a/hack/make/test b/hack/make/test index 45ffe87fbd..361f731d70 100644 --- a/hack/make/test +++ b/hack/make/test @@ -13,17 +13,34 @@ set -e bundle_test() { { date - for test_dir in $(find_test_dirs); do ( - set -x - cd $test_dir + + TESTS_FAILED=() + for test_dir in $(find_test_dirs); do + echo - # Install packages that are dependencies of the tests. - # Note: Does not run the tests. - go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS - - # Run the tests with the optional $TESTFLAGS. - go test -v -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS - ) done + if ! ( + set -x + cd $test_dir + + # Install packages that are dependencies of the tests. + # Note: Does not run the tests. + go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS + + # Run the tests with the optional $TESTFLAGS. + go test -v -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS + ); then + TESTS_FAILED+=("$test_dir") + sleep 1 # give it a second, so observers watching can take note + fi + done + + # if some tests fail, we want the bundlescript to fail, but we want to + # try running ALL the tests first, hence TESTS_FAILED + if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then + echo + echo "Test failures in: ${TESTS_FAILED[@]}" + false + fi } 2>&1 | tee $DEST/test.log } From 58f8503b7340ca34b229609a2dcb4b0fede1e638 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 21 Nov 2013 18:17:11 -0800 Subject: [PATCH 005/162] update release checklist --- hack/RELEASE-CHECKLIST.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/RELEASE-CHECKLIST.md b/hack/RELEASE-CHECKLIST.md index 7d68f9bdf5..fc7ba5cb8e 100644 --- a/hack/RELEASE-CHECKLIST.md +++ b/hack/RELEASE-CHECKLIST.md @@ -13,6 +13,7 @@ export VERSION=vXXX git checkout release git pull git checkout -b bump_$VERSION +git merge master ``` ### 2. Update CHANGELOG.md @@ -79,7 +80,7 @@ git push origin bump_$VERSION ### 8. Apply tag ```bash -git tag -a v$VERSION # Don't forget the v! +git tag -a $VERSION git push --tags ``` From 0198f8a879df8506189cf3024c4b64c9ea8f6aa6 Mon Sep 17 00:00:00 2001 From: daniel-garcia Date: Tue, 19 Nov 2013 17:27:01 -0600 Subject: [PATCH 006/162] fixes #2671, add support for bind mounting individual files in to containers, rebases of #1757 #2301 --- container.go | 37 +++++++++++++++++++++++++++++++++-- integration/container_test.go | 7 +++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/container.go b/container.go index 62f934884c..223ac811bd 100644 --- a/container.go +++ b/container.go @@ -610,6 +610,7 @@ func (container *Container) Start() (err error) { // Create the requested volumes if they don't exist for volPath := range container.Config.Volumes { volPath = path.Clean(volPath) + volIsDir := true // Skip existing volumes if _, exists := container.Volumes[volPath]; exists { continue @@ -624,6 +625,16 @@ func (container *Container) Start() (err error) { if strings.ToLower(bindMap.Mode) == "rw" { srcRW = true } + if file, err := os.Open(bindMap.SrcPath); err != nil { + return err + } else { + defer file.Close() + if stat, err := file.Stat(); err != nil { + return err + } else { + volIsDir = stat.IsDir() + } + } // Otherwise create an directory in $ROOT/volumes/ and use that } else { c, err := container.runtime.volumes.Create(nil, container, "", "", nil) @@ -640,8 +651,30 @@ func (container *Container) Start() (err error) { container.VolumesRW[volPath] = srcRW // Create the mountpoint rootVolPath := path.Join(container.RootfsPath(), volPath) - if err := os.MkdirAll(rootVolPath, 0755); err != nil { - return err + if volIsDir { + if err := os.MkdirAll(rootVolPath, 0755); err != nil { + return err + } + } + + volPath = path.Join(container.RootfsPath(), volPath) + if _, err := os.Stat(volPath); err != nil { + if os.IsNotExist(err) { + if volIsDir { + if err := os.MkdirAll(volPath, 0755); err != nil { + return err + } + } else { + if err := os.MkdirAll(path.Dir(volPath), 0755); err != nil { + return err + } + if f, err := os.OpenFile(volPath, os.O_CREATE, 0755); err != nil { + return err + } else { + f.Close() + } + } + } } // Do not copy or change permissions if we are mounting from the host diff --git a/integration/container_test.go b/integration/container_test.go index e23e3b4985..b0719d30a3 100644 --- a/integration/container_test.go +++ b/integration/container_test.go @@ -1257,6 +1257,13 @@ func TestBindMounts(t *testing.T) { if _, err := runContainer(eng, r, []string{"-v", fmt.Sprintf("%s:.", tmpDir), "_", "ls", "."}, nil); err == nil { t.Fatal("Container bind mounted illegal directory") } + + // test mount a file + runContainer(eng, r, []string{"-v", fmt.Sprintf("%s/holla:/tmp/holla:rw", tmpDir), "_", "sh", "-c", "echo -n 'yotta' > /tmp/holla"}, t) + content := readFile(path.Join(tmpDir, "holla"), t) // Will fail if the file doesn't exist + if content != "yotta" { + t.Fatal("Container failed to write to bind mount file") + } } // Test that -volumes-from supports both read-only mounts From 37e00831691567e30365c9e9ced5aa90a70f6221 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 22 Nov 2013 11:39:40 -0700 Subject: [PATCH 007/162] Add a few more small RELEASE-CHECKLIST tweaks, fixes, and improvements --- hack/RELEASE-CHECKLIST.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/hack/RELEASE-CHECKLIST.md b/hack/RELEASE-CHECKLIST.md index fc7ba5cb8e..8723d3c567 100644 --- a/hack/RELEASE-CHECKLIST.md +++ b/hack/RELEASE-CHECKLIST.md @@ -5,7 +5,6 @@ So you're in charge of a Docker release? Cool. Here's what to do. If your experience deviates from this document, please document the changes to keep it up-to-date. - ### 1. Pull from master and create a release branch ```bash @@ -13,7 +12,7 @@ export VERSION=vXXX git checkout release git pull git checkout -b bump_$VERSION -git merge master +git merge origin/master ``` ### 2. Update CHANGELOG.md @@ -55,10 +54,14 @@ EXAMPLES: ### 3. Change the contents of the VERSION file +```bash +echo ${VERSION#v} > VERSION +``` + ### 4. Run all tests ```bash -docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh test +docker run -privileged docker hack/make.sh test ``` ### 5. Test the docs @@ -80,8 +83,8 @@ git push origin bump_$VERSION ### 8. Apply tag ```bash -git tag -a $VERSION -git push --tags +git tag -a $VERSION -m $VERSION bump_$VERSION +git push origin $VERSION ``` Merging the pull request to the release branch will automatically @@ -92,6 +95,9 @@ documentation releases, see ``docs/README.md`` ### 9. Go to github to merge the bump_$VERSION into release +Don't forget to push that pretty blue button to delete the leftover +branch afterwards! + ### 10. Publish binaries To run this you will need access to the release credentials. @@ -108,17 +114,19 @@ docker run \ -e AWS_ACCESS_KEY=$(cat ~/.aws/access_key) \ -e AWS_SECRET_KEY=$(cat ~/.aws/secret_key) \ -e GPG_PASSPHRASE=supersecretsesame \ - -privileged -lxc-conf=lxc.aa_profile=unconfined \ - -t -i \ + -i -t -privileged \ docker \ hack/release.sh ``` -It will build and upload the binaries on the specified bucket (you should -use test.docker.io for general testing, and once everything is fine, -switch to get.docker.io). +It will run the test suite one more time, build the binaries and packages, +and upload to the specified bucket (you should use test.docker.io for +general testing, and once everything is fine, switch to get.docker.io). - -### 11. Rejoice! +### 11. Rejoice and Evangelize! Congratulations! You're done. + +Go forth and announce the glad tidings of the new release in `#docker`, +`#docker-dev`, on the [mailing list](https://groups.google.com/forum/#!forum/docker-dev), +and on Twitter! From 96b5be9dd9606a9bfbf0fdbe98bcaf8b6e77e4b1 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 14 Nov 2013 15:19:31 +1000 Subject: [PATCH 008/162] add more searchable info to the error message when ADD tries to go outside the context --- buildfile.go | 2 +- integration/buildfile_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildfile.go b/buildfile.go index ce157302f6..b643af509d 100644 --- a/buildfile.go +++ b/buildfile.go @@ -288,7 +288,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { destPath = destPath + "/" } if !strings.HasPrefix(origPath, b.context) { - return fmt.Errorf("Forbidden path: %s", origPath) + return fmt.Errorf("Forbidden path outside the build context: %s (%s)", orig, origPath) } fi, err := os.Stat(origPath) if err != nil { diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 964b58403b..20d0450a7c 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -483,7 +483,7 @@ func TestForbiddenContextPath(t *testing.T) { t.Fail() } - if err.Error() != "Forbidden path: /" { + if err.Error() != "Forbidden path outside the build context: ../../ (/)" { t.Logf("Error message is not expected: %s", err.Error()) t.Fail() } From f16c45f8b0447249074a0b41807a979a3ae326c5 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 24 Nov 2013 20:00:39 -0700 Subject: [PATCH 009/162] Add space-escaping to path parts of lxc.mount.entry lines in generated lxc.conf, allowing for spaces in mount point names Fixes #2802 --- lxc_template.go | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lxc_template.go b/lxc_template.go index 2ba2867428..2c68b7a837 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -1,6 +1,7 @@ package docker import ( + "strings" "text/template" ) @@ -31,8 +32,8 @@ lxc.rootfs = {{$ROOTFS}} {{if and .HostnamePath .HostsPath}} # enable domain name support -lxc.mount.entry = {{.HostnamePath}} {{$ROOTFS}}/etc/hostname none bind,ro 0 0 -lxc.mount.entry = {{.HostsPath}} {{$ROOTFS}}/etc/hosts none bind,ro 0 0 +lxc.mount.entry = {{escapeFstabSpaces .HostnamePath}} {{escapeFstabSpaces $ROOTFS}}/etc/hostname none bind,ro 0 0 +lxc.mount.entry = {{escapeFstabSpaces .HostsPath}} {{escapeFstabSpaces $ROOTFS}}/etc/hosts none bind,ro 0 0 {{end}} # use a dedicated pts for the container (and limit the number of pseudo terminal @@ -84,27 +85,27 @@ lxc.cgroup.devices.allow = c 10:200 rwm lxc.pivotdir = lxc_putold # WARNING: procfs is a known attack vector and should probably be disabled # if your userspace allows it. eg. see http://blog.zx2c4.com/749 -lxc.mount.entry = proc {{$ROOTFS}}/proc proc nosuid,nodev,noexec 0 0 +lxc.mount.entry = proc {{escapeFstabSpaces $ROOTFS}}/proc proc nosuid,nodev,noexec 0 0 # WARNING: sysfs is a known attack vector and should probably be disabled # if your userspace allows it. eg. see http://bit.ly/T9CkqJ -lxc.mount.entry = sysfs {{$ROOTFS}}/sys sysfs nosuid,nodev,noexec 0 0 -lxc.mount.entry = devpts {{$ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,nosuid,noexec 0 0 -#lxc.mount.entry = varrun {{$ROOTFS}}/var/run tmpfs mode=755,size=4096k,nosuid,nodev,noexec 0 0 -#lxc.mount.entry = varlock {{$ROOTFS}}/var/lock tmpfs size=1024k,nosuid,nodev,noexec 0 0 -lxc.mount.entry = shm {{$ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0 +lxc.mount.entry = sysfs {{escapeFstabSpaces $ROOTFS}}/sys sysfs nosuid,nodev,noexec 0 0 +lxc.mount.entry = devpts {{escapeFstabSpaces $ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,nosuid,noexec 0 0 +#lxc.mount.entry = varrun {{escapeFstabSpaces $ROOTFS}}/var/run tmpfs mode=755,size=4096k,nosuid,nodev,noexec 0 0 +#lxc.mount.entry = varlock {{escapeFstabSpaces $ROOTFS}}/var/lock tmpfs size=1024k,nosuid,nodev,noexec 0 0 +lxc.mount.entry = shm {{escapeFstabSpaces $ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0 # Inject dockerinit -lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/.dockerinit none bind,ro 0 0 +lxc.mount.entry = {{escapeFstabSpaces .SysInitPath}} {{escapeFstabSpaces $ROOTFS}}/.dockerinit none bind,ro 0 0 # Inject env -lxc.mount.entry = {{.EnvConfigPath}} {{$ROOTFS}}/.dockerenv none bind,ro 0 0 +lxc.mount.entry = {{escapeFstabSpaces .EnvConfigPath}} {{escapeFstabSpaces $ROOTFS}}/.dockerenv none bind,ro 0 0 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container -lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 +lxc.mount.entry = {{escapeFstabSpaces .ResolvConfPath}} {{escapeFstabSpaces $ROOTFS}}/etc/resolv.conf none bind,ro 0 0 {{if .Volumes}} {{ $rw := .VolumesRW }} {{range $virtualPath, $realPath := .Volumes}} -lxc.mount.entry = {{$realPath}} {{$ROOTFS}}/{{$virtualPath}} none bind,{{ if index $rw $virtualPath }}rw{{else}}ro{{end}} 0 0 +lxc.mount.entry = {{escapeFstabSpaces $realPath}} {{escapeFstabSpaces $ROOTFS}}/{{escapeFstabSpaces $virtualPath}} none bind,{{ if index $rw $virtualPath }}rw{{else}}ro{{end}} 0 0 {{end}} {{end}} @@ -144,6 +145,12 @@ lxc.cgroup.cpu.shares = {{.Config.CpuShares}} var LxcTemplateCompiled *template.Template +// Escape spaces in strings according to the fstab documentation, which is the +// format for "lxc.mount.entry" lines in lxc.conf. See also "man 5 fstab". +func escapeFstabSpaces(field string) string { + return strings.Replace(field, " ", "\\040", -1) +} + func getMemorySwap(config *Config) int64 { // By default, MemorySwap is set to twice the size of RAM. // If you want to omit MemorySwap, set it to `-1'. @@ -167,6 +174,7 @@ func init() { "getMemorySwap": getMemorySwap, "getHostConfig": getHostConfig, "getCapabilities": getCapabilities, + "escapeFstabSpaces": escapeFstabSpaces, } LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { From b702edadb7d63feb1f40ebfae27a4d745d4b733e Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 24 Nov 2013 20:02:06 -0700 Subject: [PATCH 010/162] Format lxc_template.go with gofmt --- lxc_template.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lxc_template.go b/lxc_template.go index 2c68b7a837..2d95a2971d 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -171,9 +171,9 @@ func getCapabilities(container *Container) *Capabilities { func init() { var err error funcMap := template.FuncMap{ - "getMemorySwap": getMemorySwap, - "getHostConfig": getHostConfig, - "getCapabilities": getCapabilities, + "getMemorySwap": getMemorySwap, + "getHostConfig": getHostConfig, + "getCapabilities": getCapabilities, "escapeFstabSpaces": escapeFstabSpaces, } LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) From 1af6ffb9bb5a1dec665a56b1bdf621fd2e2b7377 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 26 Nov 2013 00:07:59 -0700 Subject: [PATCH 011/162] Add explicit test strings for new escapeFstabSpaces function --- lxc_template_unit_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lxc_template_unit_test.go b/lxc_template_unit_test.go index ce5af1d321..ccdfec8890 100644 --- a/lxc_template_unit_test.go +++ b/lxc_template_unit_test.go @@ -100,3 +100,21 @@ func grepFile(t *testing.T, path string, pattern string) { } t.Fatalf("grepFile: pattern \"%s\" not found in \"%s\"", pattern, path) } + +func TestEscapeFstabSpaces(t *testing.T) { + var testInputs = map[string]string{ + " ": "\\040", + "": "", + "/double space": "/double\\040\\040space", + "/some long test string": "/some\\040long\\040test\\040string", + "/var/lib/docker": "/var/lib/docker", + " leading": "\\040leading", + "trailing ": "trailing\\040", + } + for in, exp := range testInputs { + if out := escapeFstabSpaces(in); exp != out { + t.Logf("Expected %s got %s", exp, out) + t.Fail() + } + } +} From 0bb2c0b1d0f651e23c949381762fe79a45c58aeb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 25 Nov 2013 23:20:36 -0800 Subject: [PATCH 012/162] fix docker run on an unknown image --- commands.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 21b2e72c28..6f0954c42c 100644 --- a/commands.go +++ b/commands.go @@ -2009,8 +2009,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { if body, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), config); err != nil { return err } - } - if err != nil { + } else if err != nil { return err } From 009024ad6439fad28ab3a9d8db6b6844bfbba8c9 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 26 Nov 2013 00:05:46 -0800 Subject: [PATCH 013/162] fix -link parsing --- commands.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 6f0954c42c..d992db2e6c 100644 --- a/commands.go +++ b/commands.go @@ -1650,11 +1650,12 @@ func (opts AttachOpts) Set(val string) error { // LinkOpts stores arguments to `docker run -link` type LinkOpts []string -func (link LinkOpts) String() string { return fmt.Sprintf("%v", []string(link)) } -func (link LinkOpts) Set(val string) error { +func (link *LinkOpts) String() string { return fmt.Sprintf("%v", []string(*link)) } +func (link *LinkOpts) Set(val string) error { if _, err := parseLink(val); err != nil { return err } + *link = append(*link, val) return nil } @@ -1760,7 +1761,7 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") - cmd.Var(flLinks, "link", "Add link to another container (name:alias)") + cmd.Var(&flLinks, "link", "Add link to another container (name:alias)") cmd.Var(&flPublish, "p", fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", PortSpecTemplateFormat)) cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host") From 9c15322894152095539e41623ef33ec0ebdc6c3f Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Tue, 26 Nov 2013 11:18:50 +0100 Subject: [PATCH 014/162] Fix the 'but is not' typo. --- runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime.go b/runtime.go index 66df7a2489..f58be836bd 100644 --- a/runtime.go +++ b/runtime.go @@ -159,7 +159,7 @@ func (runtime *Runtime) Register(container *Container) error { return err } if !strings.Contains(string(output), "RUNNING") { - utils.Debugf("Container %s was supposed to be running be is not.", container.ID) + utils.Debugf("Container %s was supposed to be running but is not.", container.ID) if runtime.config.AutoRestart { utils.Debugf("Restarting") container.State.SetGhost(false) From 82674372946b431f93add255e137f82822eecfc1 Mon Sep 17 00:00:00 2001 From: Yurii Rashkovskii Date: Tue, 26 Nov 2013 06:54:46 -0800 Subject: [PATCH 015/162] Fix command line help for docker save `docker save IMAGE DESTINATION` is not what `docker save` expects --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index d992db2e6c..f298e41d1b 100644 --- a/commands.go +++ b/commands.go @@ -2195,7 +2195,7 @@ func (cli *DockerCli) CmdCp(args ...string) error { } func (cli *DockerCli) CmdSave(args ...string) error { - cmd := cli.Subcmd("save", "IMAGE DESTINATION", "Save an image to a tar archive") + cmd := cli.Subcmd("save", "IMAGE", "Save an image to a tar archive (streamed to stdout)") if err := cmd.Parse(args); err != nil { return err } From 76c71260f13af3005c2407e2a19a422741b97117 Mon Sep 17 00:00:00 2001 From: Nicolas Kaiser Date: Tue, 26 Nov 2013 16:13:39 +0100 Subject: [PATCH 016/162] fix typo in CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4024bf2634..dbcbab6695 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ your branch before submitting a pull request. Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as -well as a clean docmuent build. See ``docs/README.md`` for more +well as a clean documentation build. See ``docs/README.md`` for more information on building the docs and how docs get released. Write clean code. Universally formatted code promotes ease of writing, reading, From ab3a83c617ec5bfafee117ea0b6e92ce78bd46e5 Mon Sep 17 00:00:00 2001 From: Martijn van Oosterhout Date: Tue, 26 Nov 2013 16:03:36 +0100 Subject: [PATCH 017/162] Add mkseccomp.pl, helper script to make seccomp profiles. --- contrib/mkseccomp.pl | 77 +++++++ contrib/mkseccomp.sample | 444 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 521 insertions(+) create mode 100755 contrib/mkseccomp.pl create mode 100644 contrib/mkseccomp.sample diff --git a/contrib/mkseccomp.pl b/contrib/mkseccomp.pl new file mode 100755 index 0000000000..44088f952c --- /dev/null +++ b/contrib/mkseccomp.pl @@ -0,0 +1,77 @@ +#!/usr/bin/perl +# +# A simple helper script to help people build seccomp profiles for +# Docker/LXC. The goal is mostly to reduce the attack surface to the +# kernel, by restricting access to rarely used, recently added or not used +# syscalls. +# +# This script processes one or more files which contain the list of system +# calls to be allowed. See mkseccomp.sample for more information how you +# can configure the list of syscalls. When run, this script produces output +# which, when stored in a file, can be passed to docker as follows: +# +# docker run -lxc-conf="lxc.seccomp=$file" +# +# The included sample file shows how to cut about a quarter of all syscalls, +# which affecting most applications. +# +# For specific situations it is possible to reduce the list further. By +# reducing the list to just those syscalls required by a certain application +# you can make it difficult for unknown/unexpected code to run. +# +# Run this script as follows: +# +# ./mkseccomp.pl < mkseccomp.sample >syscalls.list +# or +# ./mkseccomp.pl mkseccomp.sample >syscalls.list +# +# Multiple files can be specified, in which case the lists of syscalls are +# combined. +# +# By Martijn van Oosterhout Nov 2013 + +# How it works: +# +# This program basically spawns two processes to form a chain like: +# +# | cpp | + +use strict; +use warnings; + +if( -t ) { + print STDERR "Helper script to make seccomp filters for Docker/LXC.\n"; + print STDERR "Usage: mkseccomp.pl [files...]\n"; + exit 1; +} + +my $pid = open(my $in, "-|") // die "Couldn't fork1 ($!)\n"; + +if($pid == 0) { # Child + $pid = open(my $out, "|-") // die "Couldn't fork2 ($!)\n"; + + if($pid == 0) { # Child, which execs cpp + exec "cpp" or die "Couldn't exec cpp ($!)\n"; + exit 1; + } + + # Process the DATA section and output to cpp + print $out "#include \n"; + while(<>) { + if(/^\w/) { + print $out "__NR_$_"; + } + } + close $out; + exit 0; + +} + +# Print header and then process output from cpp. +print "1\n"; +print "whitelist\n"; + +while(<$in>) { + print if( /^[0-9]/ ); +} + diff --git a/contrib/mkseccomp.sample b/contrib/mkseccomp.sample new file mode 100644 index 0000000000..25bf4822dc --- /dev/null +++ b/contrib/mkseccomp.sample @@ -0,0 +1,444 @@ +/* This sample file is an example for mkseccomp.pl to produce a seccomp file + * which restricts syscalls that are only useful for an admin but allows the + * vast majority of normal userspace programs to run normally. + * + * The format of this file is one line per syscall. This is then processed + * and passed to 'cpp' to convert the names to numbers using whatever is + * correct for your platform. As such C-style comments are permitted. Note + * this also means that C preprocessor macros are also allowed. So it is + * possible to create groups surrounded by #ifdef/#endif and control their + * inclusion via #define (not #include). + * + * Syscalls that don't exist on your architecture are silently filtered out. + * Syscalls marked with (*) are required for a container to spawn a bash + * shell successfully (not necessarily full featured). Listing the same + * syscall multiple times is no problem. + * + * If you want to make a list specifically for one application the easiest + * way is to run the application under strace, like so: + * + * $ strace -f -q -c -o strace.out application args... + * + * Once you have a reasonable sample of the execution of the program, exit + * it. The file strace.out will have a summary of the syscalls used. Copy + * that list into this file, comment out everything else except the starred + * syscalls (which you need for the container to start) and you're done. + * + * To get the list of syscalls from the strace output this works well for + * me + * + * $ cut -c52 < strace.out + * + * This sample list was compiled as a combination of all the syscalls + * available on i386 and amd64 on Ubuntu Precise, as such it may not contain + * everything and not everything may be relevent for your system. This + * shouldn't be a problem. + */ + +// Filesystem/File descriptor related +access // (*) +chdir // (*) +chmod +chown +chown32 +close // (*) +creat +dup // (*) +dup2 // (*) +dup3 +epoll_create +epoll_create1 +epoll_ctl +epoll_ctl_old +epoll_pwait +epoll_wait +epoll_wait_old +eventfd +eventfd2 +faccessat // (*) +fadvise64 +fadvise64_64 +fallocate +fanotify_init +fanotify_mark +ioctl // (*) +fchdir +fchmod +fchmodat +fchown +fchown32 +fchownat +fcntl // (*) +fcntl64 +fdatasync +fgetxattr +flistxattr +flock +fremovexattr +fsetxattr +fstat // (*) +fstat64 +fstatat64 +fstatfs +fstatfs64 +fsync +ftruncate +ftruncate64 +getcwd // (*) +getdents // (*) +getdents64 +getxattr +inotify_add_watch +inotify_init +inotify_init1 +inotify_rm_watch +io_cancel +io_destroy +io_getevents +io_setup +io_submit +lchown +lchown32 +lgetxattr +link +linkat +listxattr +llistxattr +llseek +_llseek +lremovexattr +lseek // (*) +lsetxattr +lstat +lstat64 +mkdir +mkdirat +mknod +mknodat +newfstatat +_newselect +oldfstat +oldlstat +oldolduname +oldstat +olduname +oldwait4 +open // (*) +openat // (*) +pipe // (*) +pipe2 +poll +ppoll +pread64 +preadv +futimesat +pselect6 +pwrite64 +pwritev +read // (*) +readahead +readdir +readlink +readlinkat +readv +removexattr +rename +renameat +rmdir +select +sendfile +sendfile64 +setxattr +splice +stat // (*) +stat64 +statfs // (*) +statfs64 +symlink +symlinkat +sync +sync_file_range +sync_file_range2 +syncfs +tee +truncate +truncate64 +umask +unlink +unlinkat +ustat +utime +utimensat +utimes +write // (*) +writev + +// Network related +accept +accept4 +bind // (*) +connect // (*) +getpeername +getsockname // (*) +getsockopt +listen +recv +recvfrom // (*) +recvmmsg +recvmsg +send +sendmmsg +sendmsg +sendto // (*) +setsockopt +shutdown +socket // (*) +socketcall +socketpair + +// Signal related +pause +rt_sigaction // (*) +rt_sigpending +rt_sigprocmask // (*) +rt_sigqueueinfo +rt_sigreturn // (*) +rt_sigsuspend +rt_sigtimedwait +rt_tgsigqueueinfo +sigaction +sigaltstack // (*) +signal +signalfd +signalfd4 +sigpending +sigprocmask +sigreturn +sigsuspend + +// Other needed POSIX +alarm +brk // (*) +clock_adjtime +clock_getres +clock_gettime +clock_nanosleep +//clock_settime +gettimeofday +nanosleep +nice +sysinfo +syslog +time +timer_create +timer_delete +timerfd_create +timerfd_gettime +timerfd_settime +timer_getoverrun +timer_gettime +timer_settime +times +uname // (*) + +// Memory control +madvise +mbind +mincore +mlock +mlockall +mmap // (*) +mmap2 +mprotect // (*) +mremap +msync +munlock +munlockall +munmap // (*) +remap_file_pages +set_mempolicy +vmsplice + +// Process control +capget +//capset +clone // (*) +execve // (*) +exit // (*) +exit_group // (*) +fork +getcpu +getpgid +getpgrp // (*) +getpid // (*) +getppid // (*) +getpriority +getresgid +getresgid32 +getresuid +getresuid32 +getrlimit // (*) +getrusage +getsid +getuid // (*) +getuid32 +getegid // (*) +getegid32 +geteuid // (*) +geteuid32 +getgid // (*) +getgid32 +getgroups +getgroups32 +getitimer +get_mempolicy +kill +//personality +prctl +prlimit64 +sched_getaffinity +sched_getparam +sched_get_priority_max +sched_get_priority_min +sched_getscheduler +sched_rr_get_interval +//sched_setaffinity +//sched_setparam +//sched_setscheduler +sched_yield +setfsgid +setfsgid32 +setfsuid +setfsuid32 +setgid +setgid32 +setgroups +setgroups32 +setitimer +setpgid // (*) +setpriority +setregid +setregid32 +setresgid +setresgid32 +setresuid +setresuid32 +setreuid +setreuid32 +setrlimit +setsid +setuid +setuid32 +ugetrlimit +vfork +wait4 // (*) +waitid +waitpid + +// IPC +ipc +mq_getsetattr +mq_notify +mq_open +mq_timedreceive +mq_timedsend +mq_unlink +msgctl +msgget +msgrcv +msgsnd +semctl +semget +semop +semtimedop +shmat +shmctl +shmdt +shmget + +// Linux specific, mostly needed for thread-related stuff +arch_prctl // (*) +get_robust_list +get_thread_area +gettid +futex // (*) +restart_syscall // (*) +set_robust_list // (*) +set_thread_area +set_tid_address // (*) +tgkill +tkill + +// Admin syscalls, these are blocked +//acct +//adjtimex +//bdflush +//chroot +//create_module +//delete_module +//get_kernel_syms // Obsolete +//idle // Obsolete +//init_module +//ioperm +//iopl +//ioprio_get +//ioprio_set +//kexec_load +//lookup_dcookie // oprofile only? +//migrate_pages // NUMA +//modify_ldt +//mount +//move_pages // NUMA +//name_to_handle_at // NFS server +//nfsservctl // NFS server +//open_by_handle_at // NFS server +//perf_event_open +//pivot_root +//process_vm_readv // For debugger +//process_vm_writev // For debugger +//ptrace // For debugger +//query_module +//quotactl +//reboot +//setdomainname +//sethostname +//setns +//settimeofday +//sgetmask // Obsolete +//ssetmask // Obsolete +//stime +//swapoff +//swapon +//_sysctl +//sysfs +//sys_setaltroot +//umount +//umount2 +//unshare +//uselib +//vhangup +//vm86 +//vm86old + +// Kernel key management +//add_key +//keyctl +//request_key + +// Unimplemented +//afs_syscall +//break +//ftime +//getpmsg +//gtty +//lock +//madvise1 +//mpx +//prof +//profil +//putpmsg +//security +//stty +//tuxcall +//ulimit +//vserver From 8398abf0dcda8fa7818b69355af9b99ab4e8bdfc Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 26 Nov 2013 09:36:46 -0700 Subject: [PATCH 018/162] Fix CHANGELOG: we ended up not merging the btrfs driver for last night's release --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75dd094bd4..bdd91eaddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ #### Notable features since 0.6.0 -* Storage drivers: choose from aufs, device mapper, vfs or btrfs. -* Standard Linux support: docker now runs on unmodified linux kernels and all major distributions. +* Storage drivers: choose from aufs, device-mapper, or vfs. +* Standard Linux support: docker now runs on unmodified Linux kernels and all major distributions. * Links: compose complex software stacks by connecting containers to each other. * Container naming: organize your containers by giving them memorable names. * Advanced port redirects: specify port redirects per interface, or keep sensitive ports private. From 1ba11384bf82f824b0efbab31aaca439cfba1b4f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 17:46:06 +0000 Subject: [PATCH 019/162] Refactor Opts --- buildfile.go | 2 +- commands.go | 117 ++++++++++------------------------------- docker/docker.go | 24 +++++---- opts.go | 132 +++++++++++++++++++++++++++++++++++++++++++++++ utils.go | 6 +-- utils/utils.go | 12 ----- 6 files changed, 178 insertions(+), 115 deletions(-) create mode 100644 opts.go diff --git a/buildfile.go b/buildfile.go index 4ac8e75bd4..bb84f54027 100644 --- a/buildfile.go +++ b/buildfile.go @@ -241,7 +241,7 @@ func (b *buildFile) CmdVolume(args string) error { volume = []string{args} } if b.config.Volumes == nil { - b.config.Volumes = PathOpts{} + b.config.Volumes = map[string]struct{}{} } for _, v := range volume { b.config.Volumes[v] = struct{}{} diff --git a/commands.go b/commands.go index d992db2e6c..ba91b14923 100644 --- a/commands.go +++ b/commands.go @@ -23,7 +23,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "reflect" "regexp" "runtime" @@ -1635,54 +1634,6 @@ func (cli *DockerCli) CmdSearch(args ...string) error { // Ports type - Used to parse multiple -p flags type ports []int -// AttachOpts stores arguments to 'docker run -a', eg. which streams to attach to -type AttachOpts map[string]bool - -func (opts AttachOpts) String() string { return fmt.Sprintf("%v", map[string]bool(opts)) } -func (opts AttachOpts) Set(val string) error { - if val != "stdin" && val != "stdout" && val != "stderr" { - return fmt.Errorf("Unsupported stream name: %s", val) - } - opts[val] = true - return nil -} - -// LinkOpts stores arguments to `docker run -link` -type LinkOpts []string - -func (link *LinkOpts) String() string { return fmt.Sprintf("%v", []string(*link)) } -func (link *LinkOpts) Set(val string) error { - if _, err := parseLink(val); err != nil { - return err - } - *link = append(*link, val) - return nil -} - -// PathOpts stores a unique set of absolute paths -type PathOpts map[string]struct{} - -func (opts PathOpts) String() string { return fmt.Sprintf("%v", map[string]struct{}(opts)) } -func (opts PathOpts) Set(val string) error { - var containerPath string - - splited := strings.SplitN(val, ":", 2) - if len(splited) == 1 { - containerPath = splited[0] - val = filepath.Clean(splited[0]) - } else { - containerPath = splited[1] - val = fmt.Sprintf("%s:%s", splited[0], filepath.Clean(splited[1])) - } - - if !filepath.IsAbs(containerPath) { - utils.Debugf("%s is not an absolute path", containerPath) - return fmt.Errorf("%s is not an absolute path", containerPath) - } - opts[val] = struct{}{} - return nil -} - func (cli *DockerCli) CmdTag(args ...string) error { cmd := cli.Subcmd("tag", "[OPTIONS] IMAGE REPOSITORY[:TAG]", "Tag an image into a repository") force := cmd.Bool("f", false, "Force") @@ -1728,16 +1679,16 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) { var ( // FIXME: use utils.ListOpts for attach and volumes? - flAttach = AttachOpts{} - flVolumes = PathOpts{} - flLinks = LinkOpts{} + flAttach = NewListOpts(ValidateAttach) + flVolumes = NewListOpts(ValidatePath) + flLinks = NewListOpts(ValidateLink) + flEnv = NewListOpts(ValidateEnv) - flPublish utils.ListOpts - flExpose utils.ListOpts - flEnv utils.ListOpts - flDns utils.ListOpts - flVolumesFrom utils.ListOpts - flLxcOpts utils.ListOpts + flPublish ListOpts + flExpose ListOpts + flDns ListOpts + flVolumesFrom ListOpts + flLxcOpts ListOpts flAutoRemove = cmd.Bool("rm", false, "Automatically remove the container when it exits (incompatible with -d)") flDetach = cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id") @@ -1759,13 +1710,13 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co _ = cmd.String("name", "", "Assign a name to the container") ) - cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") - cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") + cmd.Var(&flAttach, "a", "Attach to stdin, stdout or stderr.") + cmd.Var(&flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") cmd.Var(&flLinks, "link", "Add link to another container (name:alias)") + cmd.Var(&flEnv, "e", "Set environment variables") cmd.Var(&flPublish, "p", fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", PortSpecTemplateFormat)) cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host") - cmd.Var(&flEnv, "e", "Set environment variables") cmd.Var(&flDns, "dns", "Set custom dns servers") cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)") cmd.Var(&flLxcOpts, "lxc-conf", "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"") @@ -1780,7 +1731,7 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co } // Validate input params - if *flDetach && len(flAttach) > 0 { + if *flDetach && flAttach.Len() > 0 { return nil, nil, cmd, ErrConflictAttachDetach } if *flWorkingDir != "" && !path.IsAbs(*flWorkingDir) { @@ -1791,7 +1742,7 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co } // If neither -d or -a are set, attach to everything by default - if len(flAttach) == 0 && !*flDetach { + if flAttach.Len() == 0 && !*flDetach { if !*flDetach { flAttach.Set("stdout") flAttach.Set("stderr") @@ -1801,17 +1752,6 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co } } - var envs []string - for _, env := range flEnv { - arr := strings.Split(env, "=") - if len(arr) > 1 { - envs = append(envs, env) - } else { - v := os.Getenv(env) - envs = append(envs, env+"="+v) - } - } - var flMemory int64 if *flMemoryString != "" { parsedMemory, err := utils.RAMInBytes(*flMemoryString) @@ -1823,16 +1763,15 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co var binds []string // add any bind targets to the list of container volumes - for bind := range flVolumes { - arr := strings.Split(bind, ":") - if len(arr) > 1 { + for bind := range flVolumes.GetMap() { + if arr := strings.Split(bind, ":"); len(arr) > 1 { if arr[0] == "/" { return nil, nil, cmd, fmt.Errorf("Invalid bind mount: source can't be '/'") } dstDir := arr[1] - flVolumes[dstDir] = struct{}{} + flVolumes.Set(dstDir) binds = append(binds, bind) - delete(flVolumes, bind) + flVolumes.Delete(bind) } } @@ -1867,13 +1806,13 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co domainname = parts[1] } - ports, portBindings, err := parsePortSpecs(flPublish) + ports, portBindings, err := parsePortSpecs(flPublish.GetAll()) if err != nil { return nil, nil, cmd, err } // Merge in exposed ports to the map of published ports - for _, e := range flExpose { + for _, e := range flExpose.GetAll() { if strings.Contains(e, ":") { return nil, nil, cmd, fmt.Errorf("Invalid port format for -expose: %s", e) } @@ -1894,15 +1833,15 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co OpenStdin: *flStdin, Memory: flMemory, CpuShares: *flCpuShares, - AttachStdin: flAttach["stdin"], - AttachStdout: flAttach["stdout"], - AttachStderr: flAttach["stderr"], - Env: envs, + AttachStdin: flAttach.Get("stdin"), + AttachStdout: flAttach.Get("stdout"), + AttachStderr: flAttach.Get("stderr"), + Env: flEnv.GetAll(), Cmd: runCmd, - Dns: flDns, + Dns: flDns.GetAll(), Image: image, - Volumes: flVolumes, - VolumesFrom: strings.Join(flVolumesFrom, ","), + Volumes: flVolumes.GetMap(), + VolumesFrom: strings.Join(flVolumesFrom.GetAll(), ","), Entrypoint: entrypoint, WorkingDir: *flWorkingDir, } @@ -1913,7 +1852,7 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co LxcConf: lxcConf, Privileged: *flPrivileged, PortBindings: portBindings, - Links: flLinks, + Links: flLinks.GetAll(), PublishAllPorts: *flPublishAll, } diff --git a/docker/docker.go b/docker/docker.go index 83df5c2171..3f9be0a92a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -33,26 +33,30 @@ func main() { flRoot := flag.String("g", "/var/lib/docker", "Path to use as the root of the docker runtime") flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS headers in the remote API") flDns := flag.String("dns", "", "Force docker to use specific DNS servers") - flHosts := utils.ListOpts{fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)} - flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") + flEnableIptables := flag.Bool("iptables", true, "Disable docker's addition of iptables rules") flDefaultIp := flag.String("ip", "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm := flag.Bool("icc", true, "Enable inter-container communication") flGraphDriver := flag.String("s", "", "Force the docker runtime to use a specific storage driver") + flHosts := docker.ListOpts{} + flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) + flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") + flag.Parse() if *flVersion { showVersion() return } - if len(flHosts) > 1 { - flHosts = flHosts[1:] //trick to display a nice default value in the usage - } - for i, flHost := range flHosts { + // if flHosts.Len() > 1 { + // flHosts = flHosts[1:] //trick to display a nice default value in the usage + // } + for _, flHost := range flHosts.GetAll() { host, err := utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost) if err == nil { - flHosts[i] = host + flHosts.Set(host) + flHosts.Delete(flHost) } else { log.Fatal(err) } @@ -88,16 +92,16 @@ func main() { log.Fatal(err) } // Serve api - job = eng.Job("serveapi", flHosts...) + job = eng.Job("serveapi", flHosts.GetAll()...) job.SetenvBool("Logging", true) if err := job.Run(); err != nil { log.Fatal(err) } } else { - if len(flHosts) > 1 { + if flHosts.Len() > 1 { log.Fatal("Please specify only one -H") } - protoAddrParts := strings.SplitN(flHosts[0], "://", 2) + protoAddrParts := strings.SplitN(flHosts.GetAll()[0], "://", 2) if err := docker.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { if sterr, ok := err.(*utils.StatusError); ok { os.Exit(sterr.Status) diff --git a/opts.go b/opts.go new file mode 100644 index 0000000000..826091bea6 --- /dev/null +++ b/opts.go @@ -0,0 +1,132 @@ +package docker + +import ( + "fmt" + "github.com/dotcloud/docker/utils" + "os" + "path/filepath" + "strings" +) + +// ListOpts type +type ListOpts struct { + values []string + validator ValidatorFctType +} + +func NewListOpts(validator ValidatorFctType) ListOpts { + return ListOpts{ + validator: validator, + } +} + +func (opts *ListOpts) String() string { + return fmt.Sprintf("%v", []string(opts.values)) +} + +// Set validates if needed the input value and add it to the +// internal slice. +func (opts *ListOpts) Set(value string) error { + if opts.validator != nil { + v, err := opts.validator(value) + if err != nil { + return err + } + value = v + } + opts.values = append(opts.values, value) + return nil +} + +// Delete remove the given element from the slice. +func (opts *ListOpts) Delete(key string) { + for i, k := range opts.values { + if k == key { + opts.values = append(opts.values[:i], opts.values[i+1:]...) + return + } + } +} + +// GetMap returns the content of values in a map in order to avoid +// duplicates. +// FIXME: can we remove this? +func (opts *ListOpts) GetMap() map[string]struct{} { + ret := make(map[string]struct{}) + for _, k := range opts.values { + ret[k] = struct{}{} + } + return ret +} + +// GetAll returns the values' slice. +// FIXME: Can we remove this? +func (opts *ListOpts) GetAll() []string { + return opts.values +} + +// Get checks the existence of the given key. +func (opts *ListOpts) Get(key string) bool { + for _, k := range opts.values { + if k == key { + return true + } + } + return false +} + +// Len returns the amount of element in the slice. +func (opts *ListOpts) Len() int { + return len(opts.values) +} + +// Validators +type ValidatorFctType func(val string) (string, error) + +func ValidateAttach(val string) (string, error) { + if val != "stdin" && val != "stdout" && val != "stderr" { + return val, fmt.Errorf("Unsupported stream name: %s", val) + } + return val, nil +} + +func ValidateLink(val string) (string, error) { + if _, err := parseLink(val); err != nil { + return val, err + } + return val, nil +} + +func ValidatePath(val string) (string, error) { + var containerPath string + + splited := strings.SplitN(val, ":", 2) + if len(splited) == 1 { + containerPath = splited[0] + val = filepath.Clean(splited[0]) + } else { + containerPath = splited[1] + val = fmt.Sprintf("%s:%s", splited[0], filepath.Clean(splited[1])) + } + + if !filepath.IsAbs(containerPath) { + return val, fmt.Errorf("%s is not an absolute path", containerPath) + } + return val, nil +} + +func ValidateEnv(val string) (string, error) { + arr := strings.Split(val, "=") + if len(arr) > 1 { + return val, nil + } + return fmt.Sprintf("%s=%s", val, os.Getenv(val)), nil +} + +func ValidateHost(val string) (string, error) { + host, err := utils.ParseHost(DEFAULTHTTPHOST, DEFAULTHTTPPORT, val) + if err != nil { + return val, err + } + return host, nil +} diff --git a/utils.go b/utils.go index f62e46104c..d723affbfe 100644 --- a/utils.go +++ b/utils.go @@ -215,9 +215,9 @@ func MergeConfig(userConf, imageConf *Config) error { return nil } -func parseLxcConfOpts(opts utils.ListOpts) ([]KeyValuePair, error) { - out := make([]KeyValuePair, len(opts)) - for i, o := range opts { +func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) { + out := make([]KeyValuePair, opts.Len()) + for i, o := range opts.GetAll() { k, v, err := parseLxcOpt(o) if err != nil { return nil, err diff --git a/utils/utils.go b/utils/utils.go index cfdc73bb2e..5eeaec6659 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -34,18 +34,6 @@ type Fataler interface { Fatal(args ...interface{}) } -// ListOpts type -type ListOpts []string - -func (opts *ListOpts) String() string { - return fmt.Sprint(*opts) -} - -func (opts *ListOpts) Set(value string) error { - *opts = append(*opts, value) - return nil -} - // Go is a basic promise implementation: it wraps calls a function in a goroutine, // and returns a channel which will later return the function's return value. func Go(f func() error) chan error { From 1beb5005d11b8fadd04e51c4a1341779d26a475d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 17:47:58 +0000 Subject: [PATCH 020/162] Format main() --- docker/docker.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 3f9be0a92a..43b93dc2f4 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -23,23 +23,24 @@ func main() { sysinit.SysInit() return } - // FIXME: Switch d and D ? (to be more sshd like) - flVersion := flag.Bool("v", false, "Print version information and quit") - flDaemon := flag.Bool("d", false, "Enable daemon mode") - flDebug := flag.Bool("D", false, "Enable debug mode") - flAutoRestart := flag.Bool("r", true, "Restart previously running containers") - bridgeName := flag.String("b", "", "Attach containers to a pre-existing network bridge; use 'none' to disable container networking") - pidfile := flag.String("p", "/var/run/docker.pid", "Path to use for daemon PID file") - flRoot := flag.String("g", "/var/lib/docker", "Path to use as the root of the docker runtime") - flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS headers in the remote API") - flDns := flag.String("dns", "", "Force docker to use specific DNS servers") - flEnableIptables := flag.Bool("iptables", true, "Disable docker's addition of iptables rules") - flDefaultIp := flag.String("ip", "0.0.0.0", "Default IP address to use when binding container ports") - flInterContainerComm := flag.Bool("icc", true, "Enable inter-container communication") - flGraphDriver := flag.String("s", "", "Force the docker runtime to use a specific storage driver") + var ( + flVersion = flag.Bool("v", false, "Print version information and quit") + flDaemon = flag.Bool("d", false, "Enable daemon mode") + flDebug = flag.Bool("D", false, "Enable debug mode") + flAutoRestart = flag.Bool("r", true, "Restart previously running containers") + bridgeName = flag.String("b", "", "Attach containers to a pre-existing network bridge; use 'none' to disable container networking") + pidfile = flag.String("p", "/var/run/docker.pid", "Path to use for daemon PID file") + flRoot = flag.String("g", "/var/lib/docker", "Path to use as the root of the docker runtime") + flEnableCors = flag.Bool("api-enable-cors", false, "Enable CORS headers in the remote API") + flDns = flag.String("dns", "", "Force docker to use specific DNS servers") + flEnableIptables = flag.Bool("iptables", true, "Disable docker's addition of iptables rules") + flDefaultIp = flag.String("ip", "0.0.0.0", "Default IP address to use when binding container ports") + flInterContainerComm = flag.Bool("icc", true, "Enable inter-container communication") + flGraphDriver = flag.String("s", "", "Force the docker runtime to use a specific storage driver") - flHosts := docker.ListOpts{} + flHosts = docker.ListOpts{} + ) flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") From 5e3f6e7023fbd0cfd1233a99c332801755340cfb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 17:50:43 +0000 Subject: [PATCH 021/162] Change the default Host affectation to not rely on slice --- docker/docker.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 43b93dc2f4..2e1f2beed8 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -38,10 +38,8 @@ func main() { flDefaultIp = flag.String("ip", "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm = flag.Bool("icc", true, "Enable inter-container communication") flGraphDriver = flag.String("s", "", "Force the docker runtime to use a specific storage driver") - - flHosts = docker.ListOpts{} + flHosts = docker.NewListOpts(docker.ValidateHost) ) - flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") flag.Parse() @@ -50,17 +48,9 @@ func main() { showVersion() return } - // if flHosts.Len() > 1 { - // flHosts = flHosts[1:] //trick to display a nice default value in the usage - // } - for _, flHost := range flHosts.GetAll() { - host, err := utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost) - if err == nil { - flHosts.Set(host) - flHosts.Delete(flHost) - } else { - log.Fatal(err) - } + if flHosts.Len() == 0 { + // If we do not have a host, default to unix socket + flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) } if *flDebug { From c707c587c1fbd8e68f1e9a3c64ee0aca764461eb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 20:16:16 +0000 Subject: [PATCH 022/162] Add ParseRun unit tests --- commands_unit_test.go | 78 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 commands_unit_test.go diff --git a/commands_unit_test.go b/commands_unit_test.go new file mode 100644 index 0000000000..b7e6e894b8 --- /dev/null +++ b/commands_unit_test.go @@ -0,0 +1,78 @@ +package docker + +import ( + "strings" + "testing" +) + +func parse(t *testing.T, args string) (*Config, *HostConfig, error) { + config, hostConfig, _, err := ParseRun(strings.Split(args+" ubuntu bash", " "), nil) + return config, hostConfig, err +} + +func mustParse(t *testing.T, args string) (*Config, *HostConfig) { + config, hostConfig, err := parse(t, args) + if err != nil { + t.Fatal(err) + } + return config, hostConfig +} + +func TestParseRunLinks(t *testing.T) { + if _, hostConfig := mustParse(t, "-link a:b"); len(hostConfig.Links) == 0 || hostConfig.Links[0] != "a:b" { + t.Fatalf("Error parsing links. Expected []string{\"a:b\"}, received: %v", hostConfig.Links) + } + if _, hostConfig := mustParse(t, "-link a:b -link c:d"); len(hostConfig.Links) < 2 || hostConfig.Links[0] != "a:b" || hostConfig.Links[1] != "c:d" { + t.Fatalf("Error parsing links. Expected []string{\"a:b\", \"c:d\"}, received: %v", hostConfig.Links) + } + if _, hostConfig := mustParse(t, ""); len(hostConfig.Links) != 0 { + t.Fatalf("Error parsing links. No link expected, received: %v", hostConfig.Links) + } + + if _, _, err := parse(t, "-link a"); err == nil { + t.Fatalf("Error parsing links. `-link a` should be an error but is not") + } + if _, _, err := parse(t, "-link"); err == nil { + t.Fatalf("Error parsing links. `-link` should be an error but is not") + } +} + +func TestParseRunAttach(t *testing.T) { + if config, _ := mustParse(t, "-a stdin"); !config.AttachStdin || config.AttachStdout || config.AttachStderr { + t.Fatalf("Error parsing attach flags. Expect only Stdin enabled. Received: in: %v, out: %v, err: %v", config.AttachStdin, config.AttachStdout, config.AttachStderr) + } + if config, _ := mustParse(t, "-a stdin -a stdout"); !config.AttachStdin || !config.AttachStdout || config.AttachStderr { + t.Fatalf("Error parsing attach flags. Expect only Stdin and Stdout enabled. Received: in: %v, out: %v, err: %v", config.AttachStdin, config.AttachStdout, config.AttachStderr) + } + if config, _ := mustParse(t, "-a stdin -a stdout -a stderr"); !config.AttachStdin || !config.AttachStdout || !config.AttachStderr { + t.Fatalf("Error parsing attach flags. Expect all attach enabled. Received: in: %v, out: %v, err: %v", config.AttachStdin, config.AttachStdout, config.AttachStderr) + } + if config, _ := mustParse(t, ""); config.AttachStdin || !config.AttachStdout || !config.AttachStderr { + t.Fatalf("Error parsing attach flags. Expect Stdin disabled. Received: in: %v, out: %v, err: %v", config.AttachStdin, config.AttachStdout, config.AttachStderr) + } + + if _, _, err := parse(t, "-a"); err == nil { + t.Fatalf("Error parsing attach flags, `-a` should be an error but is not") + } + if _, _, err := parse(t, "-a invalid"); err == nil { + t.Fatalf("Error parsing attach flags, `-a invalid` should be an error but is not") + } + if _, _, err := parse(t, "-a invalid -a stdout"); err == nil { + t.Fatalf("Error parsing attach flags, `-a stdout -a invalid` should be an error but is not") + } + if _, _, err := parse(t, "-a stdout -a stderr -d"); err == nil { + t.Fatalf("Error parsing attach flags, `-a stdout -a stderr -d` should be an error but is not") + } + if _, _, err := parse(t, "-a stdin -d"); err == nil { + t.Fatalf("Error parsing attach flags, `-a stdin -d` should be an error but is not") + } + if _, _, err := parse(t, "-a stdout -d"); err == nil { + t.Fatalf("Error parsing attach flags, `-a stdout -d` should be an error but is not") + } + if _, _, err := parse(t, "-a stderr -d"); err == nil { + t.Fatalf("Error parsing attach flags, `-a stderr -d` should be an error but is not") + } + if _, _, err := parse(t, "-d -rm"); err == nil { + t.Fatalf("Error parsing attach flags, `-d -rm` should be an error but is not") + } +} From c7661f40b6c6a23e5fe2090a94bd9fa6521242d7 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 23:00:44 +0000 Subject: [PATCH 023/162] Make volumes opts more strict --- commands.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index d992db2e6c..db752447f0 100644 --- a/commands.go +++ b/commands.go @@ -1666,8 +1666,11 @@ func (opts PathOpts) String() string { return fmt.Sprintf("%v", map[string]struc func (opts PathOpts) Set(val string) error { var containerPath string - splited := strings.SplitN(val, ":", 2) - if len(splited) == 1 { + if strings.Count(val, ":") > 2 { + return fmt.Errorf("bad format for volumes: %s", val) + } + + if splited := strings.SplitN(val, ":", 2); len(splited) == 1 { containerPath = splited[0] val = filepath.Clean(splited[0]) } else { @@ -1680,6 +1683,7 @@ func (opts PathOpts) Set(val string) error { return fmt.Errorf("%s is not an absolute path", containerPath) } opts[val] = struct{}{} + return nil } From 462e30dcbdc5010abe21b206bcd4be0c8bc71be6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 26 Nov 2013 23:03:50 +0000 Subject: [PATCH 024/162] Add parseRun volume unit tests --- commands_unit_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/commands_unit_test.go b/commands_unit_test.go index b7e6e894b8..2eac5ce60d 100644 --- a/commands_unit_test.go +++ b/commands_unit_test.go @@ -76,3 +76,82 @@ func TestParseRunAttach(t *testing.T) { t.Fatalf("Error parsing attach flags, `-d -rm` should be an error but is not") } } + +func TestParseRunVolumes(t *testing.T) { + if config, hostConfig := mustParse(t, "-v /tmp"); hostConfig.Binds != nil { + t.Fatalf("Error parsing volume flags, `-v /tmp` should not mount-bind anything. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/tmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /tmp` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, "-v /tmp -v /var"); hostConfig.Binds != nil { + t.Fatalf("Error parsing volume flags, `-v /tmp -v /var` should not mount-bind anything. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/tmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /tmp` is missing from volumes. Recevied %v", config.Volumes) + } else if _, exists := config.Volumes["/var"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /var` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, "-v /hostTmp:/containerTmp"); hostConfig.Binds == nil || hostConfig.Binds[0] != "/hostTmp:/containerTmp" { + t.Fatalf("Error parsing volume flags, `-v /hostTmp:/containerTmp` should mount-bind /hostTmp into /containeTmp. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/containerTmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /tmp` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, "-v /hostTmp:/containerTmp -v /hostVar:/containerVar"); hostConfig.Binds == nil || hostConfig.Binds[0] != "/hostTmp:/containerTmp" || hostConfig.Binds[1] != "/hostVar:/containerVar" { + t.Fatalf("Error parsing volume flags, `-v /hostTmp:/containerTmp -v /hostVar:/containerVar` should mount-bind /hostTmp into /containeTmp and /hostVar into /hostContainer. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/containerTmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerTmp` is missing from volumes. Received %v", config.Volumes) + } else if _, exists := config.Volumes["/containerVar"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerVar` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, "-v /hostTmp:/containerTmp:ro -v /hostVar:/containerVar:rw"); hostConfig.Binds == nil || hostConfig.Binds[0] != "/hostTmp:/containerTmp:ro" || hostConfig.Binds[1] != "/hostVar:/containerVar:rw" { + t.Fatalf("Error parsing volume flags, `-v /hostTmp:/containerTmp:ro -v /hostVar:/containerVar:rw` should mount-bind /hostTmp into /containeTmp and /hostVar into /hostContainer. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/containerTmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerTmp` is missing from volumes. Received %v", config.Volumes) + } else if _, exists := config.Volumes["/containerVar"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerVar` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, "-v /hostTmp:/containerTmp -v /containerVar"); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != "/hostTmp:/containerTmp" { + t.Fatalf("Error parsing volume flags, `-v /hostTmp:/containerTmp -v /containerVar` should mount-bind only /hostTmp into /containeTmp. Received %v", hostConfig.Binds) + } else if _, exists := config.Volumes["/containerTmp"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerTmp` is missing from volumes. Received %v", config.Volumes) + } else if _, exists := config.Volumes["/containerVar"]; !exists { + t.Fatalf("Error parsing volume flags, `-v /containerVar` is missing from volumes. Received %v", config.Volumes) + } + + if config, hostConfig := mustParse(t, ""); hostConfig.Binds != nil { + t.Fatalf("Error parsing volume flags, without volume, nothing should be mount-binded. Received %v", hostConfig.Binds) + } else if len(config.Volumes) != 0 { + t.Fatalf("Error parsing volume flags, without volume, no volume should be present. Received %v", config.Volumes) + } + + mustParse(t, "-v /") + + if _, _, err := parse(t, "-v /:/"); err == nil { + t.Fatalf("Error parsing volume flags, `-v /:/` should fail but didn't") + } + if _, _, err := parse(t, "-v"); err == nil { + t.Fatalf("Error parsing volume flags, `-v` should fail but didn't") + } + if _, _, err := parse(t, "-v /tmp:"); err == nil { + t.Fatalf("Error parsing volume flags, `-v /tmp:` should fail but didn't") + } + if _, _, err := parse(t, "-v /tmp:ro"); err == nil { + t.Fatalf("Error parsing volume flags, `-v /tmp:ro` should fail but didn't") + } + if _, _, err := parse(t, "-v /tmp::"); err == nil { + t.Fatalf("Error parsing volume flags, `-v /tmp::` should fail but didn't") + } + if _, _, err := parse(t, "-v :"); err == nil { + t.Fatalf("Error parsing volume flags, `-v :` should fail but didn't") + } + if _, _, err := parse(t, "-v ::"); err == nil { + t.Fatalf("Error parsing volume flags, `-v ::` should fail but didn't") + } + if _, _, err := parse(t, "-v /tmp:/tmp:/tmp:/tmp"); err == nil { + t.Fatalf("Error parsing volume flags, `-v /tmp:/tmp:/tmp:/tmp` should fail but didn't") + } +} From d370a889c37b762a8ddf7e29a8e666fd7cedec3d Mon Sep 17 00:00:00 2001 From: dkumor Date: Tue, 26 Nov 2013 20:00:13 -0600 Subject: [PATCH 025/162] Deleted references to AUFS AUFS is no longer a dependency (both lxc-docker and lxc-docker-git are >=0.7), and the Arch kernel doesn't need to be replaced with AUFS_friendly. --- docs/sources/installation/archlinux.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/sources/installation/archlinux.rst b/docs/sources/installation/archlinux.rst index d6dc239253..d34dc6a209 100644 --- a/docs/sources/installation/archlinux.rst +++ b/docs/sources/installation/archlinux.rst @@ -30,7 +30,6 @@ either AUR package. * bridge-utils * go * iproute2 -* linux-aufs_friendly * lxc Installation @@ -41,9 +40,6 @@ The instructions here assume **yaourt** is installed. See for information on building and installing packages from the AUR if you have not done so before. -Keep in mind that if **linux-aufs_friendly** is not already installed that a -new kernel will be compiled and this can take quite a while. - :: yaourt -S lxc-docker-git @@ -52,9 +48,6 @@ new kernel will be compiled and this can take quite a while. Starting Docker --------------- -Prior to starting docker modify your bootloader to use the -**linux-aufs_friendly** kernel and reboot your system. - There is a systemd service unit created for docker. To start the docker service: :: From 1cb7b9adc6b9a4e4c8f746c6886a7229b0efda91 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 27 Nov 2013 12:58:54 +1000 Subject: [PATCH 026/162] there appears to be a slash prepended to a container name internally - don't tell the user about it in an error message, it might cause them to worry about it --- runtime.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime.go b/runtime.go index f58be836bd..a7c2659b00 100644 --- a/runtime.go +++ b/runtime.go @@ -422,7 +422,8 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin if _, err := runtime.containerGraph.Set(name, id); err != nil { if strings.HasSuffix(err.Error(), "name are not unique") { conflictingContainer, _ := runtime.GetByName(name) - return nil, nil, fmt.Errorf("Conflict, The name %s is already assigned to %s. You have to delete (or rename) that container to be able to assign %s to a container again.", name, utils.TruncateID(conflictingContainer.ID), name) + nameAsKnownByUser := strings.TrimPrefix(name, "/") + return nil, nil, fmt.Errorf("Conflict, The name %s is already assigned to %s. You have to delete (or rename) that container to be able to assign %s to a container again.", nameAsKnownByUser, utils.TruncateID(conflictingContainer.ID), nameAsKnownByUser) } return nil, nil, err } From 7b95d410926bffa6d8f42095fa8b38e01a5ac67b Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 27 Nov 2013 13:05:39 +1000 Subject: [PATCH 027/162] tell anyone that might want to ignore their editor choice's backup files that there is a better way - thanks @pnasrat --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 00d66de3ed..a40d9e067c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Docker project generated files to ignore +# if you want to ignore files created by your editor/tools, +# please consider a global .gitignore https://help.github.com/articles/ignoring-files .vagrant* bin docker/docker From b3e8ba19085732d9f633e4cbddb660f5f8dc5462 Mon Sep 17 00:00:00 2001 From: dkumor Date: Tue, 26 Nov 2013 22:07:56 -0600 Subject: [PATCH 028/162] Arch docs: Updated dependencies to match AUR The AUR packages lxc-docker and lxc-docker-git have changed their dependencies. --- docs/sources/installation/archlinux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/installation/archlinux.rst b/docs/sources/installation/archlinux.rst index d34dc6a209..e415295918 100644 --- a/docs/sources/installation/archlinux.rst +++ b/docs/sources/installation/archlinux.rst @@ -26,8 +26,8 @@ Dependencies Docker depends on several packages which are specified as dependencies in either AUR package. -* aufs3 * bridge-utils +* device-mapper * go * iproute2 * lxc From 0ff9bc1be3ae044107732c605986a0af20220134 Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Wed, 27 Nov 2013 09:10:44 +0100 Subject: [PATCH 029/162] Make sure the firewall rules are created even if the bridge interface is already created --- network.go | 58 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/network.go b/network.go index 1397de0557..8cdbc0e2b3 100644 --- a/network.go +++ b/network.go @@ -167,30 +167,6 @@ func CreateBridgeIface(config *DaemonConfig) error { return fmt.Errorf("Unable to start network bridge: %s", err) } - if config.EnableIptables { - // Enable NAT - if output, err := iptables.Raw("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr, - "!", "-d", ifaceAddr, "-j", "MASQUERADE"); err != nil { - return fmt.Errorf("Unable to enable network bridge NAT: %s", err) - } else if len(output) != 0 { - return fmt.Errorf("Error iptables postrouting: %s", output) - } - - // Accept incoming packets for existing connections - if output, err := iptables.Raw("-I", "FORWARD", "-o", config.BridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { - return fmt.Errorf("Unable to allow incoming packets: %s", err) - } else if len(output) != 0 { - return fmt.Errorf("Error iptables allow incoming: %s", output) - } - - // Accept all non-intercontainer outgoing packets - if output, err := iptables.Raw("-I", "FORWARD", "-i", config.BridgeIface, "!", "-o", config.BridgeIface, "-j", "ACCEPT"); err != nil { - return fmt.Errorf("Unable to allow outgoing packets: %s", err) - } else if len(output) != 0 { - return fmt.Errorf("Error iptables allow outgoing: %s", output) - } - - } return nil } @@ -699,6 +675,40 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { // Configure iptables for link support if config.EnableIptables { + + // Enable NAT + natArgs := []string{"POSTROUTING", "-t", "nat", "-s", addr.String(), "!", "-d", addr.String(), "-j", "MASQUERADE"} + + if !iptables.Exists(natArgs...) { + if output, err := iptables.Raw(append([]string{"-A"}, natArgs...)...); err != nil { + return nil, fmt.Errorf("Unable to enable network bridge NAT: %s", err) + } else if len(output) != 0 { + return nil, fmt.Errorf("Error iptables postrouting: %s", output) + } + } + + // Accept incoming packets for existing connections + existingArgs := []string{"FORWARD", "-o", config.BridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} + + if !iptables.Exists(existingArgs...) { + if output, err := iptables.Raw(append([]string{"-I"}, existingArgs...)...); err != nil { + return nil, fmt.Errorf("Unable to allow incoming packets: %s", err) + } else if len(output) != 0 { + return nil, fmt.Errorf("Error iptables allow incoming: %s", output) + } + } + + // Accept all non-intercontainer outgoing packets + outgoingArgs := []string{"FORWARD", "-i", config.BridgeIface, "!", "-o", config.BridgeIface, "-j", "ACCEPT"} + + if !iptables.Exists(outgoingArgs...) { + if output, err := iptables.Raw(append([]string{"-I"}, outgoingArgs...)...); err != nil { + return nil, fmt.Errorf("Unable to allow outgoing packets: %s", err) + } else if len(output) != 0 { + return nil, fmt.Errorf("Error iptables allow outgoing: %s", output) + } + } + args := []string{"FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j"} acceptArgs := append(args, "ACCEPT") dropArgs := append(args, "DROP") From ae474e05f553c7abefc8674148e2a84a417bbf64 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 27 Nov 2013 12:18:01 +0000 Subject: [PATCH 030/162] Allow multiple clients to pull the same tag simultaneously If two clients simultaneously try to pull the same tag, there was a race whereby one would succeed and the second would generate an error. Now, the second simply waits for the first to complete. --- server.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server.go b/server.go index a988d2133d..a5c91fa84e 100644 --- a/server.go +++ b/server.go @@ -985,8 +985,14 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut if err != nil { return err } - if _, err := srv.poolAdd("pull", localName+":"+tag); err != nil { - return err + + out = utils.NewWriteFlusher(out) + + if c, err := srv.poolAdd("pull", localName+":"+tag); err != nil { + // Another pull of the same repository is already taking place; just wait for it to finish + out.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", localName)) + <-c + return nil } defer srv.poolRemove("pull", localName+":"+tag) @@ -1001,7 +1007,6 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut localName = remoteName } - out = utils.NewWriteFlusher(out) err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel) if err == registry.ErrLoginRequired { return err From 45b1e8c2362b3626ccc564496e3093386bb89cd9 Mon Sep 17 00:00:00 2001 From: Bruno Bigras Date: Tue, 26 Nov 2013 16:11:08 -0500 Subject: [PATCH 031/162] Update postgresql's version in example It seems ppa:pitti/postgresql will be deprecated and only apt.postgresql.org has 9.3. --- docs/sources/examples/postgresql_service.rst | 30 +++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/sources/examples/postgresql_service.rst b/docs/sources/examples/postgresql_service.rst index 3649775485..82ca8b59ca 100644 --- a/docs/sources/examples/postgresql_service.rst +++ b/docs/sources/examples/postgresql_service.rst @@ -45,19 +45,21 @@ Install ``python-software-properties``. apt-get -y install python-software-properties apt-get -y install software-properties-common -Add Pitti's PostgreSQL repository. It contains the most recent stable release -of PostgreSQL i.e. ``9.2``. +Add PostgreSQL's repository. It contains the most recent stable release +of PostgreSQL i.e. ``9.3``. .. code-block:: bash - add-apt-repository ppa:pitti/postgresql + apt-get -y install wget + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list apt-get update -Finally, install PostgreSQL 9.2 +Finally, install PostgreSQL 9.3 .. code-block:: bash - apt-get -y install postgresql-9.2 postgresql-client-9.2 postgresql-contrib-9.2 + apt-get -y install postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 Now, create a PostgreSQL superuser role that can create databases and other roles. Following Vagrant's convention the role will be named @@ -76,14 +78,14 @@ role. Adjust PostgreSQL configuration so that remote connections to the database are possible. Make sure that inside -``/etc/postgresql/9.2/main/pg_hba.conf`` you have following line (you will need +``/etc/postgresql/9.3/main/pg_hba.conf`` you have following line (you will need to install an editor, e.g. ``apt-get install vim``): .. code-block:: bash host all all 0.0.0.0/0 md5 -Additionaly, inside ``/etc/postgresql/9.2/main/postgresql.conf`` +Additionaly, inside ``/etc/postgresql/9.3/main/postgresql.conf`` uncomment ``listen_addresses`` so it is as follows: .. code-block:: bash @@ -115,9 +117,9 @@ Finally, run PostgreSQL server via ``docker``. CONTAINER=$(sudo docker run -d -p 5432 \ -t /postgresql \ - /bin/su postgres -c '/usr/lib/postgresql/9.2/bin/postgres \ - -D /var/lib/postgresql/9.2/main \ - -c config_file=/etc/postgresql/9.2/main/postgresql.conf') + /bin/su postgres -c '/usr/lib/postgresql/9.3/bin/postgres \ + -D /var/lib/postgresql/9.3/main \ + -c config_file=/etc/postgresql/9.3/main/postgresql.conf') Connect the PostgreSQL server using ``psql`` (You will need postgres installed on the machine. For ubuntu, use something like @@ -132,7 +134,7 @@ As before, create roles or databases if needed. .. code-block:: bash - psql (9.2.4) + psql (9.3.1) Type "help" for help. docker=# CREATE DATABASE foo OWNER=docker; @@ -160,9 +162,9 @@ container starts. .. code-block:: bash sudo docker commit -run='{"Cmd": \ - ["/bin/su", "postgres", "-c", "/usr/lib/postgresql/9.2/bin/postgres -D \ - /var/lib/postgresql/9.2/main -c \ - config_file=/etc/postgresql/9.2/main/postgresql.conf"], "PortSpecs": ["5432"]}' \ + ["/bin/su", "postgres", "-c", "/usr/lib/postgresql/9.3/bin/postgres -D \ + /var/lib/postgresql/9.3/main -c \ + config_file=/etc/postgresql/9.3/main/postgresql.conf"], "PortSpecs": ["5432"]}' \ /postgresql From now on, just type ``docker run /postgresql`` and From 682a188ead44e93ac3d5cc53c708808fa7d2fe4f Mon Sep 17 00:00:00 2001 From: dkumor Date: Wed, 27 Nov 2013 10:25:30 -0600 Subject: [PATCH 032/162] Arch docs: Added lxc-docker-nightly AUR package, modified deps lxc-docker-nightly installs latest build. Removed go from dependencies, as it is not needed in lxc-docker and lxc-docker-nightly. The -git package will flag go as a dependency upon installation. --- docs/sources/installation/archlinux.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/sources/installation/archlinux.rst b/docs/sources/installation/archlinux.rst index e415295918..ae56badf4c 100644 --- a/docs/sources/installation/archlinux.rst +++ b/docs/sources/installation/archlinux.rst @@ -12,26 +12,28 @@ Arch Linux .. include:: install_unofficial.inc Installing on Arch Linux is not officially supported but can be handled via -either of the following AUR packages: +one of the following AUR packages: * `lxc-docker `_ * `lxc-docker-git `_ +* `lxc-docker-nightly `_ The lxc-docker package will install the latest tagged version of docker. The lxc-docker-git package will build from the current master branch. +The lxc-docker-nightly package will install the latest build. Dependencies ------------ Docker depends on several packages which are specified as dependencies in -either AUR package. +the AUR packages. The core dependencies are: * bridge-utils * device-mapper -* go * iproute2 * lxc + Installation ------------ @@ -42,7 +44,7 @@ done so before. :: - yaourt -S lxc-docker-git + yaourt -S lxc-docker Starting Docker From 788feab3a7de44a4749e5a861e3b6f778aa4088b Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 27 Nov 2013 16:53:36 +0000 Subject: [PATCH 033/162] Handle the case where poolAdd() gives an error for an unknown pool type --- server.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server.go b/server.go index a5c91fa84e..3641e2fdc8 100644 --- a/server.go +++ b/server.go @@ -988,11 +988,15 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut out = utils.NewWriteFlusher(out) - if c, err := srv.poolAdd("pull", localName+":"+tag); err != nil { - // Another pull of the same repository is already taking place; just wait for it to finish - out.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", localName)) - <-c - return nil + c, err := srv.poolAdd("pull", localName+":"+tag) + if err != nil { + if c != nil { + // Another pull of the same repository is already taking place; just wait for it to finish + out.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", localName)) + <-c + return nil + } + return err } defer srv.poolRemove("pull", localName+":"+tag) From 4e826e99b278ef3cc438553e958727397b4d9ee7 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Wed, 27 Nov 2013 12:55:15 -0500 Subject: [PATCH 034/162] Performance of deleteImageAndChildren. Don't walk the file system for parents each time we recurse. Fixes #2852 --- server.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 4f5e5a2cd9..3fcbdbea95 100644 --- a/server.go +++ b/server.go @@ -1406,19 +1406,15 @@ func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool) var ErrImageReferenced = errors.New("Image referenced by a repository") -func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error { +func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi, byParents map[string][]*Image) error { // If the image is referenced by a repo, do not delete if len(srv.runtime.repositories.ByID()[id]) != 0 { return ErrImageReferenced } // If the image is not referenced but has children, go recursive referenced := false - byParents, err := srv.runtime.graph.ByParent() - if err != nil { - return err - } for _, img := range byParents[id] { - if err := srv.deleteImageAndChildren(img.ID, imgs); err != nil { + if err := srv.deleteImageAndChildren(img.ID, imgs, byParents); err != nil { if err != ErrImageReferenced { return err } @@ -1430,7 +1426,7 @@ func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error { } // If the image is not referenced and has no children, remove it - byParents, err = srv.runtime.graph.ByParent() + byParents, err := srv.runtime.graph.ByParent() if err != nil { return err } @@ -1455,8 +1451,12 @@ func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error { if err != nil { return err } + byParents, err := srv.runtime.graph.ByParent() + if err != nil { + return err + } // Remove all children images - if err := srv.deleteImageAndChildren(img.Parent, imgs); err != nil { + if err := srv.deleteImageAndChildren(img.Parent, imgs, byParents); err != nil { return err } return srv.deleteImageParents(parent, imgs) @@ -1498,7 +1498,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro } } if len(srv.runtime.repositories.ByID()[img.ID]) == 0 { - if err := srv.deleteImageAndChildren(img.ID, &imgs); err != nil { + if err := srv.deleteImageAndChildren(img.ID, &imgs, nil); err != nil { if err != ErrImageReferenced { return imgs, err } From e4ae44b844a38327ee4288361376299595b1917e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 27 Nov 2013 10:41:20 -0800 Subject: [PATCH 035/162] Add instructions for opening issues on the repository --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbcbab6695..6fed5c61b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,13 @@ Want to hack on Docker? Awesome! Here are instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete. +## Reporting Issues + +When reporting [issues](https://github.com/dotcloud/docker/issues) +on Github please include your host OS ( Ubuntu 12.04, Fedora 19, etc... ) +and the output of `docker version` along with the output of `docker info` if possible. +This information will help us review and fix your issue faster. + ## Build Environment For instructions on setting up your development environment, please From d1a631cedb24c7c75affde7041d43217db64e5c0 Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Wed, 27 Nov 2013 22:00:58 +0000 Subject: [PATCH 036/162] Only tag the top-most layer, not all interim layers --- server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 3641e2fdc8..f502661e9f 100644 --- a/server.go +++ b/server.go @@ -1098,11 +1098,16 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName for _, ep := range repoData.Endpoints { out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo))) // This section can not be parallelized (each round depends on the previous one) - for _, round := range imgList { + for i, round := range imgList { // FIXME: This section can be parallelized for _, elem := range round { var pushTags func() error pushTags = func() error { + if i < (len(imgList) - 1) { + // Only tag the top layer in the repository + return nil + } + out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err From 2c27da881807b6c63ef29e8033241a17c76f5ab7 Mon Sep 17 00:00:00 2001 From: Alexis THOMAS Date: Thu, 28 Nov 2013 00:39:06 +0100 Subject: [PATCH 037/162] Restore 'save' paragraph --- docs/sources/commandline/cli.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 9635675a77..e35475ae96 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -797,7 +797,7 @@ Known Issues (kill) -link="": Remove the link instead of the actual container Known Issues (rm) -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~ * :issue:`197` indicates that ``docker kill`` may leave directories behind and make it difficult to remove the container. @@ -881,8 +881,15 @@ containers will not be deleted. -name="": Assign the specified name to the container. If no name is specific docker will generate a random name -P=false: Publish all exposed ports to the host interfaces -Examples --------- +Known Issues (run -volumes-from) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :issue:`2702`: "lxc-start: Permission denied - failed to mount" + could indicate a permissions problem with AppArmor. Please see the + issue for a workaround. + +Examples: +~~~~~~~~~ .. code-block:: bash @@ -974,16 +981,10 @@ id may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in read-only or read-write mode, respectively. By default, the volumes are mounted in the same mode (rw or ro) as the reference container. -Known Issues (run -volumes-from) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* :issue:`2702`: "lxc-start: Permission denied - failed to mount" - could indicate a permissions problem with AppArmor. Please see the - issue for a workaround. - .. _cli_save: ``save`` +--------- :: From 74c8f7af756ed03131aee051b0ccb926b77e04db Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 07:16:34 +0000 Subject: [PATCH 038/162] Refactor attach loop device in pure Go --- graphdriver/devmapper/deviceset.go | 10 +- graphdriver/devmapper/devmapper.go | 125 +++++++++++++-- graphdriver/devmapper/devmapper_wrapper.go | 178 +++++---------------- graphdriver/devmapper/sys.go | 8 +- 4 files changed, 160 insertions(+), 161 deletions(-) diff --git a/graphdriver/devmapper/deviceset.go b/graphdriver/devmapper/deviceset.go index 8c2556104d..77d865ad99 100644 --- a/graphdriver/devmapper/deviceset.go +++ b/graphdriver/devmapper/deviceset.go @@ -383,7 +383,7 @@ func (devices *DeviceSet) ResizePool(size int64) error { return fmt.Errorf("Can't shrink file") } - dataloopback := FindLoopDeviceFor(&osFile{File: datafile}) + dataloopback := FindLoopDeviceFor(datafile) if dataloopback == nil { return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) } @@ -395,7 +395,7 @@ func (devices *DeviceSet) ResizePool(size int64) error { } defer metadatafile.Close() - metadataloopback := FindLoopDeviceFor(&osFile{File: metadatafile}) + metadataloopback := FindLoopDeviceFor(metadatafile) if metadataloopback == nil { return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) } @@ -491,14 +491,14 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { if info.Exists == 0 { utils.Debugf("Pool doesn't exist. Creating it.") - dataFile, err := AttachLoopDevice(data) + dataFile, err := attachLoopDevice(data) if err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } defer dataFile.Close() - metadataFile, err := AttachLoopDevice(metadata) + metadataFile, err := attachLoopDevice(metadata) if err != nil { utils.Debugf("\n--->Err: %s\n", err) return err @@ -637,7 +637,7 @@ func (devices *DeviceSet) deactivateDevice(hash string) error { // or b) the 1 second timeout expires. func (devices *DeviceSet) waitRemove(hash string) error { utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash) - defer utils.Debugf("[deviceset %s] waitRemove END", devices.devicePrefix, hash) + defer utils.Debugf("[deviceset %s] waitRemove(%) END", devices.devicePrefix, hash) devname, err := devices.byHash(hash) if err != nil { return err diff --git a/graphdriver/devmapper/devmapper.go b/graphdriver/devmapper/devmapper.go index 89c36d45ad..305ed183d4 100644 --- a/graphdriver/devmapper/devmapper.go +++ b/graphdriver/devmapper/devmapper.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/dotcloud/docker/utils" "runtime" + "unsafe" ) type DevmapperLogger interface { @@ -177,15 +178,6 @@ func (t *Task) GetNextTarget(next uintptr) (nextPtr uintptr, start uint64, start, length, targetType, params } -func AttachLoopDevice(filename string) (*osFile, error) { - var fd int - res := DmAttachLoopDevice(filename, &fd) - if res == "" { - return nil, ErrAttachLoopbackDevice - } - return &osFile{File: osNewFile(uintptr(fd), res)}, nil -} - func getLoopbackBackingFile(file *osFile) (uint64, uint64, error) { dev, inode, err := DmGetLoopbackBackingFile(file.Fd()) if err != 0 { @@ -223,11 +215,10 @@ func FindLoopDeviceFor(file *osFile) *osFile { continue } - dev, inode, err := getLoopbackBackingFile(&osFile{File: file}) + dev, inode, err := getLoopbackBackingFile(file) if err == nil && dev == targetDevice && inode == targetInode { - return &osFile{File: file} + return file } - file.Close() } @@ -420,7 +411,7 @@ func suspendDevice(name string) error { return err } if err := task.Run(); err != nil { - return fmt.Errorf("Error running DeviceSuspend") + return fmt.Errorf("Error running DeviceSuspend: %s", err) } return nil } @@ -437,7 +428,7 @@ func resumeDevice(name string) error { } if err := task.Run(); err != nil { - return fmt.Errorf("Error running DeviceSuspend") + return fmt.Errorf("Error running DeviceResume") } UdevWait(cookie) @@ -574,3 +565,109 @@ func (devices *DeviceSet) createSnapDevice(poolName string, deviceId int, baseNa return nil } + +type LoopInfo64 struct { + loDevice uint64 /* ioctl r/o */ + loInode uint64 /* ioctl r/o */ + loRdevice uint64 /* ioctl r/o */ + loOffset uint64 + loSizelimit uint64 /* bytes, 0 == max available */ + loNumber uint32 /* ioctl r/o */ + loEncrypt_type uint32 + loEncrypt_key_size uint32 /* ioctl w/o */ + loFlags uint32 /* ioctl r/o */ + loFileName [LoNameSize]uint8 + loCryptName [LoNameSize]uint8 + loEncryptKey [LoKeySize]uint8 /* ioctl w/o */ + loInit [2]uint64 +} + +// attachLoopDevice attaches the given sparse file to the next +// available loopback device. It returns an opened *osFile. +func attachLoopDevice(filename string) (loop *osFile, err error) { + startIndex := 0 + + // Try to retrieve the next available loopback device via syscall. + // If it fails, we discard error and start loopking for a + // loopback from index 0. + if f, err := osOpenFile("/dev/loop-control", osORdOnly, 0644); err == nil { + if index, _, err := sysSyscall(sysSysIoctl, f.Fd(), LoopCtlGetFree, 0); err != 0 { + utils.Debugf("Error retrieving the next available loopback: %s", err) + } else if index > 0 { + startIndex = int(index) + } + f.Close() + } + + // Open the given sparse file (use OpenFile because Open sets O_CLOEXEC) + f, err := osOpenFile(filename, osORdWr, 0644) + if err != nil { + return nil, err + } + defer f.Close() + + var ( + target string + loopFile *osFile + ) + // Start looking for a free /dev/loop + for i := startIndex; ; { + target = fmt.Sprintf("/dev/loop%d", i) + + fi, err := osStat(target) + if err != nil { + if osIsNotExist(err) { + utils.Errorf("There are no more loopback device available.") + } + } + + // FIXME: Check here if target is a block device (in C: S_ISBLK(mode)) + if fi.IsDir() { + } + + // Open the targeted loopback (use OpenFile because Open sets O_CLOEXEC) + loopFile, err = osOpenFile(target, osORdWr, 0644) + if err != nil { + return nil, err + } + + // Try to attach to the loop file + if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopSetFd, f.Fd()); err != 0 { + loopFile.Close() + // If the error is EBUSY, then try the next loopback + if err != sysEBusy { + utils.Errorf("Cannot set up loopback device %s: %s", target, err) + return nil, err + } + } else { + // In case of success, we finished. Break the loop. + break + } + // In case of EBUSY error, the loop keep going. + } + + // This can't happen, but let's be sure + if loopFile == nil { + return nil, fmt.Errorf("Unreachable code reached! Error attaching %s to a loopback device.", filename) + } + + // Set the status of the loopback device + var loopInfo LoopInfo64 + + // Due to type incompatibility (string vs [64]uint8), we copy data + copy(loopInfo.loFileName[:], target[:]) + loopInfo.loOffset = 0 + loopInfo.loFlags = LoFlagsAutoClear + + if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopSetStatus64, uintptr(unsafe.Pointer(&loopInfo))); err != 0 { + // If the call failed, then free the loopback device + utils.Errorf("Cannot set up loopback device info: %s", err) + if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopClrFd, 0); err != 0 { + utils.Errorf("Error while cleaning up the loopback device") + } + loopFile.Close() + return nil, err + } + + return loopFile, nil +} diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 818c968bbf..0ed5bcd332 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -2,124 +2,14 @@ package devmapper /* #cgo LDFLAGS: -L. -ldevmapper -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include - -#ifndef LOOP_CTL_GET_FREE -#define LOOP_CTL_GET_FREE 0x4C82 -#endif - -// FIXME: this could easily be rewritten in go -char* attach_loop_device(const char *filename, int *loop_fd_out) -{ - struct loop_info64 loopinfo = {0}; - struct stat st; - char buf[64]; - int i, loop_fd, fd, start_index; - char* loopname; - - - *loop_fd_out = -1; - - start_index = 0; - fd = open("/dev/loop-control", O_RDONLY); - if (fd >= 0) { - start_index = ioctl(fd, LOOP_CTL_GET_FREE); - close(fd); - - if (start_index < 0) - start_index = 0; - } - - fd = open(filename, O_RDWR); - if (fd < 0) { - perror("open"); - return NULL; - } - - loop_fd = -1; - for (i = start_index ; loop_fd < 0 ; i++ ) { - if (sprintf(buf, "/dev/loop%d", i) < 0) { - close(fd); - return NULL; - } - - if (stat(buf, &st)) { - if (!S_ISBLK(st.st_mode)) { - fprintf(stderr, "[error] Loopback device %s is not a block device.\n", buf); - } else if (errno == ENOENT) { - fprintf(stderr, "[error] There are no more loopback device available.\n"); - } else { - fprintf(stderr, "[error] Unkown error trying to stat the loopback device %s (errno: %d).\n", buf, errno); - } - close(fd); - return NULL; - } - - loop_fd = open(buf, O_RDWR); - if (loop_fd < 0 && errno == ENOENT) { - fprintf(stderr, "[error] The loopback device %s does not exists.\n", buf); - close(fd); - return NULL; - } else if (loop_fd < 0) { - fprintf(stderr, "[error] Unkown error openning the loopback device %s. (errno: %d)\n", buf, errno); - continue; - } - - if (ioctl(loop_fd, LOOP_SET_FD, (void *)(size_t)fd) < 0) { - int errsv = errno; - close(loop_fd); - loop_fd = -1; - if (errsv != EBUSY) { - close(fd); - fprintf(stderr, "cannot set up loopback device %s: %s", buf, strerror(errsv)); - return NULL; - } - continue; - } - - close(fd); - - strncpy((char*)loopinfo.lo_file_name, buf, LO_NAME_SIZE); - loopinfo.lo_offset = 0; - loopinfo.lo_flags = LO_FLAGS_AUTOCLEAR; - - if (ioctl(loop_fd, LOOP_SET_STATUS64, &loopinfo) < 0) { - perror("ioctl LOOP_SET_STATUS64"); - if (ioctl(loop_fd, LOOP_CLR_FD, 0) < 0) { - perror("ioctl LOOP_CLR_FD"); - } - close(loop_fd); - fprintf (stderr, "cannot set up loopback device info"); - return (NULL); - } - - loopname = strdup(buf); - if (loopname == NULL) { - close(loop_fd); - return (NULL); - } - - *loop_fd_out = loop_fd; - return (loopname); - } - - return (NULL); -} +#include // FIXME: present only for defines, maybe we can remove it? +#include // FIXME: present only for BLKGETSIZE64, maybe we can remove it? +// FIXME: Can't we find a way to do the logging in pure Go? extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str); -static void log_cb(int level, const char *file, int line, - int dm_errno_or_class, const char *f, ...) +static void log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...) { char buffer[256]; va_list ap; @@ -135,7 +25,6 @@ static void log_with_errno_init() { dm_log_with_errno_init(log_cb); } - */ import "C" @@ -147,6 +36,19 @@ type ( CDmTask C.struct_dm_task ) +// FIXME: Make sure the values are defined in C +const ( + LoopSetFd = C.LOOP_SET_FD + LoopCtlGetFree = C.LOOP_CTL_GET_FREE + LoopSetStatus64 = C.LOOP_SET_STATUS64 + LoopClrFd = C.LOOP_CLR_FD + LoFlagsAutoClear = C.LO_FLAGS_AUTOCLEAR + LoFlagsReadOnly = C.LO_FLAGS_READ_ONLY + LoFlagsPartScan = C.LO_FLAGS_PARTSCAN + LoKeySize = C.LO_KEY_SIZE + LoNameSize = C.LO_NAME_SIZE +) + var ( DmAttachLoopDevice = dmAttachLoopDeviceFct DmGetBlockSize = dmGetBlockSizeFct @@ -185,28 +87,26 @@ func dmTaskCreateFct(taskType int) *CDmTask { } func dmTaskRunFct(task *CDmTask) int { - return int(C.dm_task_run((*C.struct_dm_task)(task))) + ret, _ := C.dm_task_run((*C.struct_dm_task)(task)) + return int(ret) } func dmTaskSetNameFct(task *CDmTask, name string) int { Cname := C.CString(name) defer free(Cname) - return int(C.dm_task_set_name((*C.struct_dm_task)(task), - Cname)) + return int(C.dm_task_set_name((*C.struct_dm_task)(task), Cname)) } func dmTaskSetMessageFct(task *CDmTask, message string) int { Cmessage := C.CString(message) defer free(Cmessage) - return int(C.dm_task_set_message((*C.struct_dm_task)(task), - Cmessage)) + return int(C.dm_task_set_message((*C.struct_dm_task)(task), Cmessage)) } func dmTaskSetSectorFct(task *CDmTask, sector uint64) int { - return int(C.dm_task_set_sector((*C.struct_dm_task)(task), - C.uint64_t(sector))) + return int(C.dm_task_set_sector((*C.struct_dm_task)(task), C.uint64_t(sector))) } func dmTaskSetCookieFct(task *CDmTask, cookie *uint, flags uint16) int { @@ -214,13 +114,11 @@ func dmTaskSetCookieFct(task *CDmTask, cookie *uint, flags uint16) int { defer func() { *cookie = uint(cCookie) }() - return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie, - C.uint16_t(flags))) + return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie, C.uint16_t(flags))) } func dmTaskSetAddNodeFct(task *CDmTask, addNode AddNodeType) int { - return int(C.dm_task_set_add_node((*C.struct_dm_task)(task), - C.dm_add_node_t(addNode))) + return int(C.dm_task_set_add_node((*C.struct_dm_task)(task), C.dm_add_node_t(addNode))) } func dmTaskSetRoFct(task *CDmTask) int { @@ -236,14 +134,12 @@ func dmTaskAddTargetFct(task *CDmTask, Cparams := C.CString(params) defer free(Cparams) - return int(C.dm_task_add_target((*C.struct_dm_task)(task), - C.uint64_t(start), C.uint64_t(size), Cttype, Cparams)) + return int(C.dm_task_add_target((*C.struct_dm_task)(task), C.uint64_t(start), C.uint64_t(size), Cttype, Cparams)) } func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { var lo64 C.struct_loop_info64 - _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, - uintptr(unsafe.Pointer(&lo64))) + _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) } @@ -287,23 +183,23 @@ func dmGetNextTargetFct(task *CDmTask, next uintptr, start, length *uint64, targ *params = C.GoString(Cparams) }() - nextp := C.dm_get_next_target((*C.struct_dm_task)(task), - unsafe.Pointer(next), &Cstart, &Clength, &CtargetType, &Cparams) + nextp := C.dm_get_next_target((*C.struct_dm_task)(task), unsafe.Pointer(next), &Cstart, &Clength, &CtargetType, &Cparams) return uintptr(nextp) } func dmAttachLoopDeviceFct(filename string, fd *int) string { - cFilename := C.CString(filename) - defer free(cFilename) + return "" + // cFilename := C.CString(filename) + // defer free(cFilename) - var cFd C.int - defer func() { - *fd = int(cFd) - }() + // var cFd C.int + // defer func() { + // *fd = int(cFd) + // }() - ret := C.attach_loop_device(cFilename, &cFd) - defer free(ret) - return C.GoString(ret) + // ret := C.attach_loop_device(cFilename, &cFd) + // defer free(ret) + // return C.GoString(ret) } func getBlockSizeFct(fd uintptr, size *uint64) sysErrno { diff --git a/graphdriver/devmapper/sys.go b/graphdriver/devmapper/sys.go index 60bafb5f6d..152e68285b 100644 --- a/graphdriver/devmapper/sys.go +++ b/graphdriver/devmapper/sys.go @@ -19,7 +19,11 @@ var ( sysCloseOnExec = syscall.CloseOnExec sysSyscall = syscall.Syscall - osOpenFile = os.OpenFile + osOpenFile = func(name string, flag int, perm os.FileMode) (*osFile, error) { + f, err := os.OpenFile(name, flag, perm) + return &osFile{File: f}, err + } + osOpen = func(name string) (*osFile, error) { f, err := os.Open(name); return &osFile{File: f}, err } osNewFile = os.NewFile osCreate = os.Create osStat = os.Stat @@ -40,7 +44,9 @@ const ( sysMsRdOnly = syscall.MS_RDONLY sysEInval = syscall.EINVAL sysSysIoctl = syscall.SYS_IOCTL + sysEBusy = syscall.EBUSY + osORdOnly = os.O_RDONLY osORdWr = os.O_RDWR osOCreate = os.O_CREATE ) From 75e9cff98ce585ccd050a4014adb99d95509c9b7 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Nov 2013 16:55:37 -0700 Subject: [PATCH 039/162] Update systemd service file with suggestions from @seblu --- contrib/init/systemd/docker.service | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/init/systemd/docker.service b/contrib/init/systemd/docker.service index bdc6522efa..aae7b6daf9 100644 --- a/contrib/init/systemd/docker.service +++ b/contrib/init/systemd/docker.service @@ -1,11 +1,9 @@ [Unit] Description=Docker Application Container Engine Documentation=http://docs.docker.io -Requires=network.target -After=multi-user.target +After=network.target [Service] -Type=simple ExecStartPre=/bin/mount --make-rprivate / ExecStart=/usr/bin/docker -d From eb528b959efc0a3924939991be45704047fd5aea Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 17:12:57 -0800 Subject: [PATCH 040/162] Move attach loop device to its own file --- graphdriver/devmapper/attachLoopback.go | 178 +++++++++++++++++++++ graphdriver/devmapper/devmapper.go | 107 ------------- graphdriver/devmapper/devmapper_wrapper.go | 60 +++---- 3 files changed, 208 insertions(+), 137 deletions(-) create mode 100644 graphdriver/devmapper/attachLoopback.go diff --git a/graphdriver/devmapper/attachLoopback.go b/graphdriver/devmapper/attachLoopback.go new file mode 100644 index 0000000000..d81a482ac4 --- /dev/null +++ b/graphdriver/devmapper/attachLoopback.go @@ -0,0 +1,178 @@ +package devmapper + +import ( + "fmt" + "github.com/dotcloud/docker/utils" + "unsafe" +) + +func ioctlLoopCtlGetFree(fd uintptr) (int, error) { + index, _, err := sysSyscall(sysSysIoctl, fd, LoopCtlGetFree, 0) + if err != 0 { + return 0, err + } + return int(index), nil +} + +func ioctlLoopSetFd(loopFd, sparseFd uintptr) error { + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetFd, sparseFd); err != 0 { + return err + } + return nil +} + +func ioctlLoopSetStatus64(loopFd uintptr, loopInfo *LoopInfo64) error { + _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetStatus64, uintptr(unsafe.Pointer(loopInfo))) + if err != 0 { + return err + } + return nil +} + +func ioctlLoopClrFd(loopFd uintptr) error { + _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopClrFd, 0) + if err != 0 { + return err + } + return nil +} + +// //func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { +// func ioctlLoopGetStatus64(loopFd uintptr) (*LoopInfo64, error) { +// var lo64 C.struct_loop_info64 +// _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) +// return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) +// } + +// func dmLoopbackSetCapacityFct(fd uintptr) sysErrno { +// _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_SET_CAPACITY, 0) +// return sysErrno(err) +// } + +// func dmGetBlockSizeFct(fd uintptr) (int64, sysErrno) { +// var size int64 +// _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) +// return size, sysErrno(err) +// } + +// func getBlockSizeFct(fd uintptr, size *uint64) sysErrno { +// _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) +// return sysErrno(err) +// } + +func getNextFreeLoopbackIndex() (int, error) { + f, err := osOpenFile("/dev/loop-control", osORdOnly, 0644) + if err != nil { + return 0, err + } + defer f.Close() + + index, err := ioctlLoopCtlGetFree(f.Fd()) + if index < 0 { + index = 0 + } + return index, err +} + +func openNextAvailableLoopback(index int, sparseFile *osFile) (loopFile *osFile, err error) { + // Start looking for a free /dev/loop + for { + target := fmt.Sprintf("/dev/loop%d", index) + index++ + + fi, err := osStat(target) + if err != nil { + if osIsNotExist(err) { + utils.Errorf("There are no more loopback device available.") + } + return nil, ErrAttachLoopbackDevice + } + + // FIXME: Check here if target is a block device (in C: S_ISBLK(mode)) + if fi.IsDir() { + } + + // Open the targeted loopback (use OpenFile because Open sets O_CLOEXEC) + loopFile, err = osOpenFile(target, osORdWr, 0644) + if err != nil { + utils.Errorf("Error openning loopback device: %s", err) + return nil, ErrAttachLoopbackDevice + } + + // Try to attach to the loop file + if err := ioctlLoopSetFd(loopFile.Fd(), sparseFile.Fd()); err != nil { + loopFile.Close() + + // If the error is EBUSY, then try the next loopback + if err != sysEBusy { + utils.Errorf("Cannot set up loopback device %s: %s", target, err) + return nil, ErrAttachLoopbackDevice + } + + // Otherwise, we keep going with the loop + continue + } + // In case of success, we finished. Break the loop. + break + } + + // This can't happen, but let's be sure + if loopFile == nil { + utils.Errorf("Unreachable code reached! Error attaching %s to a loopback device.", sparseFile.Name()) + return nil, ErrAttachLoopbackDevice + } + + return loopFile, nil +} + +func stringToLoopName(src string) [LoNameSize]uint8 { + var dst [LoNameSize]uint8 + copy(dst[:], src[:]) + return dst +} + +// attachLoopDevice attaches the given sparse file to the next +// available loopback device. It returns an opened *osFile. +func attachLoopDevice(sparseName string) (loop *osFile, err error) { + + // Try to retrieve the next available loopback device via syscall. + // If it fails, we discard error and start loopking for a + // loopback from index 0. + startIndex, err := getNextFreeLoopbackIndex() + if err != nil { + utils.Debugf("Error retrieving the next available loopback: %s", err) + } + + // Open the given sparse file (use OpenFile because Open sets O_CLOEXEC) + sparseFile, err := osOpenFile(sparseName, osORdWr, 0644) + if err != nil { + utils.Errorf("Error openning sparse file %s: %s", sparseName, err) + return nil, ErrAttachLoopbackDevice + } + defer sparseFile.Close() + + loopFile, err := openNextAvailableLoopback(startIndex, sparseFile) + if err != nil { + return nil, err + } + + // Set the status of the loopback device + loopInfo := &LoopInfo64{ + loFileName: stringToLoopName(loopFile.Name()), + loOffset: 0, + loFlags: LoFlagsAutoClear, + } + + if err := ioctlLoopSetStatus64(loopFile.Fd(), loopInfo); err != nil { + utils.Errorf("Cannot set up loopback device info: %s", err) + + // If the call failed, then free the loopback device + if err := ioctlLoopClrFd(loopFile.Fd()); err != nil { + utils.Errorf("Error while cleaning up the loopback device") + } + loopFile.Close() + return nil, ErrAttachLoopbackDevice + } + + return loopFile, nil +} diff --git a/graphdriver/devmapper/devmapper.go b/graphdriver/devmapper/devmapper.go index 305ed183d4..93faf5018d 100644 --- a/graphdriver/devmapper/devmapper.go +++ b/graphdriver/devmapper/devmapper.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/dotcloud/docker/utils" "runtime" - "unsafe" ) type DevmapperLogger interface { @@ -565,109 +564,3 @@ func (devices *DeviceSet) createSnapDevice(poolName string, deviceId int, baseNa return nil } - -type LoopInfo64 struct { - loDevice uint64 /* ioctl r/o */ - loInode uint64 /* ioctl r/o */ - loRdevice uint64 /* ioctl r/o */ - loOffset uint64 - loSizelimit uint64 /* bytes, 0 == max available */ - loNumber uint32 /* ioctl r/o */ - loEncrypt_type uint32 - loEncrypt_key_size uint32 /* ioctl w/o */ - loFlags uint32 /* ioctl r/o */ - loFileName [LoNameSize]uint8 - loCryptName [LoNameSize]uint8 - loEncryptKey [LoKeySize]uint8 /* ioctl w/o */ - loInit [2]uint64 -} - -// attachLoopDevice attaches the given sparse file to the next -// available loopback device. It returns an opened *osFile. -func attachLoopDevice(filename string) (loop *osFile, err error) { - startIndex := 0 - - // Try to retrieve the next available loopback device via syscall. - // If it fails, we discard error and start loopking for a - // loopback from index 0. - if f, err := osOpenFile("/dev/loop-control", osORdOnly, 0644); err == nil { - if index, _, err := sysSyscall(sysSysIoctl, f.Fd(), LoopCtlGetFree, 0); err != 0 { - utils.Debugf("Error retrieving the next available loopback: %s", err) - } else if index > 0 { - startIndex = int(index) - } - f.Close() - } - - // Open the given sparse file (use OpenFile because Open sets O_CLOEXEC) - f, err := osOpenFile(filename, osORdWr, 0644) - if err != nil { - return nil, err - } - defer f.Close() - - var ( - target string - loopFile *osFile - ) - // Start looking for a free /dev/loop - for i := startIndex; ; { - target = fmt.Sprintf("/dev/loop%d", i) - - fi, err := osStat(target) - if err != nil { - if osIsNotExist(err) { - utils.Errorf("There are no more loopback device available.") - } - } - - // FIXME: Check here if target is a block device (in C: S_ISBLK(mode)) - if fi.IsDir() { - } - - // Open the targeted loopback (use OpenFile because Open sets O_CLOEXEC) - loopFile, err = osOpenFile(target, osORdWr, 0644) - if err != nil { - return nil, err - } - - // Try to attach to the loop file - if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopSetFd, f.Fd()); err != 0 { - loopFile.Close() - // If the error is EBUSY, then try the next loopback - if err != sysEBusy { - utils.Errorf("Cannot set up loopback device %s: %s", target, err) - return nil, err - } - } else { - // In case of success, we finished. Break the loop. - break - } - // In case of EBUSY error, the loop keep going. - } - - // This can't happen, but let's be sure - if loopFile == nil { - return nil, fmt.Errorf("Unreachable code reached! Error attaching %s to a loopback device.", filename) - } - - // Set the status of the loopback device - var loopInfo LoopInfo64 - - // Due to type incompatibility (string vs [64]uint8), we copy data - copy(loopInfo.loFileName[:], target[:]) - loopInfo.loOffset = 0 - loopInfo.loFlags = LoFlagsAutoClear - - if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopSetStatus64, uintptr(unsafe.Pointer(&loopInfo))); err != 0 { - // If the call failed, then free the loopback device - utils.Errorf("Cannot set up loopback device info: %s", err) - if _, _, err := sysSyscall(sysSysIoctl, loopFile.Fd(), LoopClrFd, 0); err != 0 { - utils.Errorf("Error while cleaning up the loopback device") - } - loopFile.Close() - return nil, err - } - - return loopFile, nil -} diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 0ed5bcd332..24d9502b49 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -33,7 +33,23 @@ import ( ) type ( - CDmTask C.struct_dm_task + CDmTask C.struct_dm_task + CLoopInfo64 C.struct_loop_info64 + LoopInfo64 struct { + loDevice uint64 /* ioctl r/o */ + loInode uint64 /* ioctl r/o */ + loRdevice uint64 /* ioctl r/o */ + loOffset uint64 + loSizelimit uint64 /* bytes, 0 == max available */ + loNumber uint32 /* ioctl r/o */ + loEncrypt_type uint32 + loEncrypt_key_size uint32 /* ioctl w/o */ + loFlags uint32 /* ioctl r/o */ + loFileName [LoNameSize]uint8 + loCryptName [LoNameSize]uint8 + loEncryptKey [LoKeySize]uint8 /* ioctl w/o */ + loInit [2]uint64 + } ) // FIXME: Make sure the values are defined in C @@ -50,7 +66,6 @@ const ( ) var ( - DmAttachLoopDevice = dmAttachLoopDeviceFct DmGetBlockSize = dmGetBlockSizeFct DmGetLibraryVersion = dmGetLibraryVersionFct DmGetNextTarget = dmGetNextTargetFct @@ -137,23 +152,6 @@ func dmTaskAddTargetFct(task *CDmTask, return int(C.dm_task_add_target((*C.struct_dm_task)(task), C.uint64_t(start), C.uint64_t(size), Cttype, Cparams)) } -func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { - var lo64 C.struct_loop_info64 - _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) - return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) -} - -func dmLoopbackSetCapacityFct(fd uintptr) sysErrno { - _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_SET_CAPACITY, 0) - return sysErrno(err) -} - -func dmGetBlockSizeFct(fd uintptr) (int64, sysErrno) { - var size int64 - _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) - return size, sysErrno(err) -} - func dmTaskGetInfoFct(task *CDmTask, info *Info) int { Cinfo := C.struct_dm_info{} defer func() { @@ -187,19 +185,21 @@ func dmGetNextTargetFct(task *CDmTask, next uintptr, start, length *uint64, targ return uintptr(nextp) } -func dmAttachLoopDeviceFct(filename string, fd *int) string { - return "" - // cFilename := C.CString(filename) - // defer free(cFilename) +func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { + var lo64 C.struct_loop_info64 + _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) + return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) +} - // var cFd C.int - // defer func() { - // *fd = int(cFd) - // }() +func dmLoopbackSetCapacityFct(fd uintptr) sysErrno { + _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_SET_CAPACITY, 0) + return sysErrno(err) +} - // ret := C.attach_loop_device(cFilename, &cFd) - // defer free(ret) - // return C.GoString(ret) +func dmGetBlockSizeFct(fd uintptr) (int64, sysErrno) { + var size int64 + _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) + return size, sysErrno(err) } func getBlockSizeFct(fd uintptr, size *uint64) sysErrno { From 1214b8897bba2a33a7ded8779bcdc966fe1cb176 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 17:44:54 -0800 Subject: [PATCH 041/162] Extract ioctl from wrapper --- graphdriver/devmapper/attachLoopback.go | 63 ++---------------- graphdriver/devmapper/devmapper.go | 15 +++-- graphdriver/devmapper/devmapper_wrapper.go | 76 ++++++++-------------- graphdriver/devmapper/ioctl.go | 58 +++++++++++++++++ 4 files changed, 99 insertions(+), 113 deletions(-) create mode 100644 graphdriver/devmapper/ioctl.go diff --git a/graphdriver/devmapper/attachLoopback.go b/graphdriver/devmapper/attachLoopback.go index d81a482ac4..98d0bf5f5b 100644 --- a/graphdriver/devmapper/attachLoopback.go +++ b/graphdriver/devmapper/attachLoopback.go @@ -3,63 +3,14 @@ package devmapper import ( "fmt" "github.com/dotcloud/docker/utils" - "unsafe" ) -func ioctlLoopCtlGetFree(fd uintptr) (int, error) { - index, _, err := sysSyscall(sysSysIoctl, fd, LoopCtlGetFree, 0) - if err != 0 { - return 0, err - } - return int(index), nil +func stringToLoopName(src string) [LoNameSize]uint8 { + var dst [LoNameSize]uint8 + copy(dst[:], src[:]) + return dst } -func ioctlLoopSetFd(loopFd, sparseFd uintptr) error { - if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetFd, sparseFd); err != 0 { - return err - } - return nil -} - -func ioctlLoopSetStatus64(loopFd uintptr, loopInfo *LoopInfo64) error { - _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetStatus64, uintptr(unsafe.Pointer(loopInfo))) - if err != 0 { - return err - } - return nil -} - -func ioctlLoopClrFd(loopFd uintptr) error { - _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopClrFd, 0) - if err != 0 { - return err - } - return nil -} - -// //func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { -// func ioctlLoopGetStatus64(loopFd uintptr) (*LoopInfo64, error) { -// var lo64 C.struct_loop_info64 -// _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) -// return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) -// } - -// func dmLoopbackSetCapacityFct(fd uintptr) sysErrno { -// _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_SET_CAPACITY, 0) -// return sysErrno(err) -// } - -// func dmGetBlockSizeFct(fd uintptr) (int64, sysErrno) { -// var size int64 -// _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) -// return size, sysErrno(err) -// } - -// func getBlockSizeFct(fd uintptr, size *uint64) sysErrno { -// _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) -// return sysErrno(err) -// } - func getNextFreeLoopbackIndex() (int, error) { f, err := osOpenFile("/dev/loop-control", osORdOnly, 0644) if err != nil { @@ -125,12 +76,6 @@ func openNextAvailableLoopback(index int, sparseFile *osFile) (loopFile *osFile, return loopFile, nil } -func stringToLoopName(src string) [LoNameSize]uint8 { - var dst [LoNameSize]uint8 - copy(dst[:], src[:]) - return dst -} - // attachLoopDevice attaches the given sparse file to the next // available loopback device. It returns an opened *osFile. func attachLoopDevice(sparseName string) (loop *osFile, err error) { diff --git a/graphdriver/devmapper/devmapper.go b/graphdriver/devmapper/devmapper.go index 93faf5018d..db41ed6c25 100644 --- a/graphdriver/devmapper/devmapper.go +++ b/graphdriver/devmapper/devmapper.go @@ -178,15 +178,17 @@ func (t *Task) GetNextTarget(next uintptr) (nextPtr uintptr, start uint64, } func getLoopbackBackingFile(file *osFile) (uint64, uint64, error) { - dev, inode, err := DmGetLoopbackBackingFile(file.Fd()) - if err != 0 { + loopInfo, err := ioctlLoopGetStatus64(file.Fd()) + if err != nil { + utils.Errorf("Error get loopback backing file: %s\n", err) return 0, 0, ErrGetLoopbackBackingFile } - return dev, inode, nil + return loopInfo.loDevice, loopInfo.loInode, nil } func LoopbackSetCapacity(file *osFile) error { - if err := DmLoopbackSetCapacity(file.Fd()); err != 0 { + if err := ioctlLoopSetCapacity(file.Fd(), 0); err != nil { + utils.Errorf("Error loopbackSetCapacity: %s", err) return ErrLoopbackSetCapacity } return nil @@ -276,8 +278,9 @@ func RemoveDevice(name string) error { } func GetBlockDeviceSize(file *osFile) (uint64, error) { - size, errno := DmGetBlockSize(file.Fd()) - if size == -1 || errno != 0 { + size, err := ioctlBlkGetSize64(file.Fd()) + if err != nil { + utils.Errorf("Error getblockdevicesize: %s", err) return 0, ErrGetBlockSize } return uint64(size), nil diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 24d9502b49..1b636d392f 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -33,7 +33,8 @@ import ( ) type ( - CDmTask C.struct_dm_task + CDmTask C.struct_dm_task + CLoopInfo64 C.struct_loop_info64 LoopInfo64 struct { loDevice uint64 /* ioctl r/o */ @@ -54,39 +55,40 @@ type ( // FIXME: Make sure the values are defined in C const ( - LoopSetFd = C.LOOP_SET_FD - LoopCtlGetFree = C.LOOP_CTL_GET_FREE - LoopSetStatus64 = C.LOOP_SET_STATUS64 - LoopClrFd = C.LOOP_CLR_FD + LoopSetFd = C.LOOP_SET_FD + LoopCtlGetFree = C.LOOP_CTL_GET_FREE + LoopGetStatus64 = C.LOOP_GET_STATUS64 + LoopSetStatus64 = C.LOOP_SET_STATUS64 + LoopClrFd = C.LOOP_CLR_FD + LoopSetCapacity = C.LOOP_SET_CAPACITY + LoFlagsAutoClear = C.LO_FLAGS_AUTOCLEAR LoFlagsReadOnly = C.LO_FLAGS_READ_ONLY LoFlagsPartScan = C.LO_FLAGS_PARTSCAN LoKeySize = C.LO_KEY_SIZE LoNameSize = C.LO_NAME_SIZE + + BlkGetSize64 = C.BLKGETSIZE64 ) var ( - DmGetBlockSize = dmGetBlockSizeFct - DmGetLibraryVersion = dmGetLibraryVersionFct - DmGetNextTarget = dmGetNextTargetFct - DmLogInitVerbose = dmLogInitVerboseFct - DmSetDevDir = dmSetDevDirFct - DmTaskAddTarget = dmTaskAddTargetFct - DmTaskCreate = dmTaskCreateFct - DmTaskDestroy = dmTaskDestroyFct - DmTaskGetInfo = dmTaskGetInfoFct - DmTaskRun = dmTaskRunFct - DmTaskSetAddNode = dmTaskSetAddNodeFct - DmTaskSetCookie = dmTaskSetCookieFct - DmTaskSetMessage = dmTaskSetMessageFct - DmTaskSetName = dmTaskSetNameFct - DmTaskSetRo = dmTaskSetRoFct - DmTaskSetSector = dmTaskSetSectorFct - DmUdevWait = dmUdevWaitFct - GetBlockSize = getBlockSizeFct - LogWithErrnoInit = logWithErrnoInitFct - DmGetLoopbackBackingFile = dmGetLoopbackBackingFileFct - DmLoopbackSetCapacity = dmLoopbackSetCapacityFct + DmGetLibraryVersion = dmGetLibraryVersionFct + DmGetNextTarget = dmGetNextTargetFct + DmLogInitVerbose = dmLogInitVerboseFct + DmSetDevDir = dmSetDevDirFct + DmTaskAddTarget = dmTaskAddTargetFct + DmTaskCreate = dmTaskCreateFct + DmTaskDestroy = dmTaskDestroyFct + DmTaskGetInfo = dmTaskGetInfoFct + DmTaskRun = dmTaskRunFct + DmTaskSetAddNode = dmTaskSetAddNodeFct + DmTaskSetCookie = dmTaskSetCookieFct + DmTaskSetMessage = dmTaskSetMessageFct + DmTaskSetName = dmTaskSetNameFct + DmTaskSetRo = dmTaskSetRoFct + DmTaskSetSector = dmTaskSetSectorFct + DmUdevWait = dmUdevWaitFct + LogWithErrnoInit = logWithErrnoInitFct ) func free(p *C.char) { @@ -185,28 +187,6 @@ func dmGetNextTargetFct(task *CDmTask, next uintptr, start, length *uint64, targ return uintptr(nextp) } -func dmGetLoopbackBackingFileFct(fd uintptr) (uint64, uint64, sysErrno) { - var lo64 C.struct_loop_info64 - _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_GET_STATUS64, uintptr(unsafe.Pointer(&lo64))) - return uint64(lo64.lo_device), uint64(lo64.lo_inode), sysErrno(err) -} - -func dmLoopbackSetCapacityFct(fd uintptr) sysErrno { - _, _, err := sysSyscall(sysSysIoctl, fd, C.LOOP_SET_CAPACITY, 0) - return sysErrno(err) -} - -func dmGetBlockSizeFct(fd uintptr) (int64, sysErrno) { - var size int64 - _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) - return size, sysErrno(err) -} - -func getBlockSizeFct(fd uintptr, size *uint64) sysErrno { - _, _, err := sysSyscall(sysSysIoctl, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))) - return sysErrno(err) -} - func dmUdevWaitFct(cookie uint) int { return int(C.dm_udev_wait(C.uint32_t(cookie))) } diff --git a/graphdriver/devmapper/ioctl.go b/graphdriver/devmapper/ioctl.go new file mode 100644 index 0000000000..65c6c1a512 --- /dev/null +++ b/graphdriver/devmapper/ioctl.go @@ -0,0 +1,58 @@ +package devmapper + +import ( + "unsafe" +) + +func ioctlLoopCtlGetFree(fd uintptr) (int, error) { + index, _, err := sysSyscall(sysSysIoctl, fd, LoopCtlGetFree, 0) + if err != 0 { + return 0, err + } + return int(index), nil +} + +func ioctlLoopSetFd(loopFd, sparseFd uintptr) error { + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetFd, sparseFd); err != 0 { + return err + } + return nil +} + +func ioctlLoopSetStatus64(loopFd uintptr, loopInfo *LoopInfo64) error { + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetStatus64, uintptr(unsafe.Pointer(loopInfo))); err != 0 { + return err + } + return nil +} + +func ioctlLoopClrFd(loopFd uintptr) error { + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopClrFd, 0); err != 0 { + return err + } + return nil +} + +func ioctlLoopGetStatus64(loopFd uintptr) (*LoopInfo64, error) { + loopInfo := &LoopInfo64{} + + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopGetStatus64, uintptr(unsafe.Pointer(loopInfo))); err != 0 { + return nil, err + } + return loopInfo, nil +} + +func ioctlLoopSetCapacity(loopFd uintptr, value int) error { + if _, _, err := sysSyscall(sysSysIoctl, loopFd, LoopSetCapacity, uintptr(value)); err != 0 { + return err + } + return nil +} + +func ioctlBlkGetSize64(fd uintptr) (int64, error) { + var size int64 + if _, _, err := sysSyscall(sysSysIoctl, fd, BlkGetSize64, uintptr(unsafe.Pointer(&size))); err != 0 { + return 0, err + } + return size, nil +} From 8a5d927a5337b1c9a0c8478c4e522ce0ded09b85 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 18:21:17 -0800 Subject: [PATCH 042/162] Check if the target loopback is a block device --- graphdriver/devmapper/attachLoopback.go | 5 +++-- graphdriver/devmapper/sys.go | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/graphdriver/devmapper/attachLoopback.go b/graphdriver/devmapper/attachLoopback.go index 98d0bf5f5b..0d556d81c2 100644 --- a/graphdriver/devmapper/attachLoopback.go +++ b/graphdriver/devmapper/attachLoopback.go @@ -39,8 +39,9 @@ func openNextAvailableLoopback(index int, sparseFile *osFile) (loopFile *osFile, return nil, ErrAttachLoopbackDevice } - // FIXME: Check here if target is a block device (in C: S_ISBLK(mode)) - if fi.IsDir() { + if fi.Mode()&osModeDevice != osModeDevice { + utils.Errorf("Loopback device %s is not a block device.", target) + continue } // Open the targeted loopback (use OpenFile because Open sets O_CLOEXEC) diff --git a/graphdriver/devmapper/sys.go b/graphdriver/devmapper/sys.go index 152e68285b..9cf124a9a5 100644 --- a/graphdriver/devmapper/sys.go +++ b/graphdriver/devmapper/sys.go @@ -46,9 +46,10 @@ const ( sysSysIoctl = syscall.SYS_IOCTL sysEBusy = syscall.EBUSY - osORdOnly = os.O_RDONLY - osORdWr = os.O_RDWR - osOCreate = os.O_CREATE + osORdOnly = os.O_RDONLY + osORdWr = os.O_RDWR + osOCreate = os.O_CREATE + osModeDevice = os.ModeDevice ) func toSysStatT(i interface{}) *sysStatT { From 8b2a7e35c35f894dca0795a4fde9ec0cfe04ce43 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Thu, 28 Nov 2013 00:22:47 -0200 Subject: [PATCH 043/162] Move syscall.Stats logic to os specific file. related to #2909. --- archive/changes.go | 2 +- archive/diff.go | 6 ++++-- archive/stat_darwin.go | 11 +++++++++++ archive/stat_linux.go | 11 +++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 archive/stat_darwin.go create mode 100644 archive/stat_linux.go diff --git a/archive/changes.go b/archive/changes.go index 83bdcae7cf..a4076fc0ad 100644 --- a/archive/changes.go +++ b/archive/changes.go @@ -181,7 +181,7 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { oldStat.Rdev != newStat.Rdev || // Don't look at size for dirs, its not a good measure of change (oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) || - oldStat.Mtim != newStat.Mtim { + getLastModification(oldStat) != getLastModification(newStat) { change := Change{ Path: newChild.path(), Kind: ChangeModify, diff --git a/archive/diff.go b/archive/diff.go index 58d30466e3..f44991ecb5 100644 --- a/archive/diff.go +++ b/archive/diff.go @@ -83,8 +83,10 @@ func ApplyLayer(dest string, layer Archive) error { } for k, v := range modifiedDirs { - aTime := time.Unix(v.Atim.Unix()) - mTime := time.Unix(v.Mtim.Unix()) + lastAccess := getLastAccess(v) + lastModification := getLastModification(v) + aTime := time.Unix(lastAccess.Unix()) + mTime := time.Unix(lastModification.Unix()) if err := os.Chtimes(k, aTime, mTime); err != nil { return err diff --git a/archive/stat_darwin.go b/archive/stat_darwin.go new file mode 100644 index 0000000000..53ae9dee2f --- /dev/null +++ b/archive/stat_darwin.go @@ -0,0 +1,11 @@ +package archive + +import "syscall" + +func getLastAccess(stat *syscall.Stat_t) syscall.Timespec { + return stat.Atimespec +} + +func getLastModification(stat *syscall.Stat_t) syscall.Timespec { + return stat.Mtimespec +} diff --git a/archive/stat_linux.go b/archive/stat_linux.go new file mode 100644 index 0000000000..50b4627c4a --- /dev/null +++ b/archive/stat_linux.go @@ -0,0 +1,11 @@ +package archive + +import "syscall" + +func getLastAccess(stat *syscall.Stat_t) syscall.Timespec { + return stat.Atim +} + +func getLastModification(stat *syscall.Stat_t) syscall.Timespec { + return stat.Mtim +} From 24c03b2d938d383dd7669dc8faad7e2110a11173 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 19:12:51 -0800 Subject: [PATCH 044/162] Make devicemapper linux-only --- graphdriver/devmapper/attachLoopback.go | 2 ++ graphdriver/devmapper/deviceset.go | 2 ++ graphdriver/devmapper/devmapper.go | 2 ++ graphdriver/devmapper/devmapper_log.go | 2 ++ graphdriver/devmapper/devmapper_test.go | 2 ++ graphdriver/devmapper/devmapper_wrapper.go | 2 ++ graphdriver/devmapper/driver.go | 2 ++ graphdriver/devmapper/driver_test.go | 2 ++ graphdriver/devmapper/ioctl.go | 2 ++ graphdriver/devmapper/mount.go | 2 ++ graphdriver/devmapper/sys.go | 2 ++ 11 files changed, 22 insertions(+) diff --git a/graphdriver/devmapper/attachLoopback.go b/graphdriver/devmapper/attachLoopback.go index 0d556d81c2..57c9a1efa7 100644 --- a/graphdriver/devmapper/attachLoopback.go +++ b/graphdriver/devmapper/attachLoopback.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/deviceset.go b/graphdriver/devmapper/deviceset.go index 77d865ad99..f04cbdc2bd 100644 --- a/graphdriver/devmapper/deviceset.go +++ b/graphdriver/devmapper/deviceset.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/devmapper.go b/graphdriver/devmapper/devmapper.go index db41ed6c25..dfbdf385d7 100644 --- a/graphdriver/devmapper/devmapper.go +++ b/graphdriver/devmapper/devmapper.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/devmapper_log.go b/graphdriver/devmapper/devmapper_log.go index 1f95eb7bac..8d54ad4e3a 100644 --- a/graphdriver/devmapper/devmapper_log.go +++ b/graphdriver/devmapper/devmapper_log.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import "C" diff --git a/graphdriver/devmapper/devmapper_test.go b/graphdriver/devmapper/devmapper_test.go index ce22864361..a43e32e059 100644 --- a/graphdriver/devmapper/devmapper_test.go +++ b/graphdriver/devmapper/devmapper_test.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 1b636d392f..df95a41e13 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper /* diff --git a/graphdriver/devmapper/driver.go b/graphdriver/devmapper/driver.go index b08d5768ef..10ac172562 100644 --- a/graphdriver/devmapper/driver.go +++ b/graphdriver/devmapper/driver.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/driver_test.go b/graphdriver/devmapper/driver_test.go index 3204575dd9..bc443a787a 100644 --- a/graphdriver/devmapper/driver_test.go +++ b/graphdriver/devmapper/driver_test.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/ioctl.go b/graphdriver/devmapper/ioctl.go index 65c6c1a512..448d2d5a50 100644 --- a/graphdriver/devmapper/ioctl.go +++ b/graphdriver/devmapper/ioctl.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/mount.go b/graphdriver/devmapper/mount.go index 7a07fff1e8..d0050484bf 100644 --- a/graphdriver/devmapper/mount.go +++ b/graphdriver/devmapper/mount.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( diff --git a/graphdriver/devmapper/sys.go b/graphdriver/devmapper/sys.go index 9cf124a9a5..ea33e600bc 100644 --- a/graphdriver/devmapper/sys.go +++ b/graphdriver/devmapper/sys.go @@ -1,3 +1,5 @@ +// +build linux + package devmapper import ( From 74ea136a49e78442c8ec14307bac8e171f5dc224 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 27 Nov 2013 19:23:48 -0800 Subject: [PATCH 045/162] Move reflink to os dependent file. OSX docker client fully functionnal. --- reflink_copy_darwin.go | 14 +++++++++++ reflink_copy_linux.go | 53 ++++++++++++++++++++++++++++++++++++++++++ utils.go | 42 --------------------------------- 3 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 reflink_copy_darwin.go create mode 100644 reflink_copy_linux.go diff --git a/reflink_copy_darwin.go b/reflink_copy_darwin.go new file mode 100644 index 0000000000..3f3147db0f --- /dev/null +++ b/reflink_copy_darwin.go @@ -0,0 +1,14 @@ +package docker + +import ( + "os" + "io" +) + +func CopyFile(dstFile, srcFile *os.File) error { + // No BTRFS reflink suppport, Fall back to normal copy + + // FIXME: Check the return of Copy and compare with dstFile.Stat().Size + _, err := io.Copy(dstFile, srcFile) + return err +} diff --git a/reflink_copy_linux.go b/reflink_copy_linux.go new file mode 100644 index 0000000000..8aae3abf3d --- /dev/null +++ b/reflink_copy_linux.go @@ -0,0 +1,53 @@ +package docker + +// FIXME: This could be easily rewritten in pure Go + +/* +#include +#include +#include + +// See linux.git/fs/btrfs/ioctl.h +#define BTRFS_IOCTL_MAGIC 0x94 +#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int) + +int +btrfs_reflink(int fd_out, int fd_in) +{ + int res; + res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in); + if (res < 0) + return errno; + return 0; +} + +*/ +import "C" + +import ( + "os" + "io" + "syscall" +) + +// FIXME: Move this to btrfs package? + +func BtrfsReflink(fd_out, fd_in uintptr) error { + res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in)) + if res != 0 { + return syscall.Errno(res) + } + return nil +} + +func CopyFile(dstFile, srcFile *os.File) error { + err := BtrfsReflink(dstFile.Fd(), srcFile.Fd()) + if err == nil { + return nil + } + + // Fall back to normal copy + // FIXME: Check the return of Copy and compare with dstFile.Stat().Size + _, err = io.Copy(dstFile, srcFile) + return err +} diff --git a/utils.go b/utils.go index f62e46104c..ddef841124 100644 --- a/utils.go +++ b/utils.go @@ -1,37 +1,13 @@ package docker -/* -#include -#include -#include - -// See linux.git/fs/btrfs/ioctl.h -#define BTRFS_IOCTL_MAGIC 0x94 -#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int) - -int -btrfs_reflink(int fd_out, int fd_in) -{ - int res; - res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in); - if (res < 0) - return errno; - return 0; -} - -*/ -import "C" import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/namesgenerator" "github.com/dotcloud/docker/utils" - "io" "io/ioutil" - "os" "strconv" "strings" - "syscall" ) type Change struct { @@ -346,13 +322,6 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error { return nil } -func BtrfsReflink(fd_out, fd_in uintptr) error { - res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in)) - if res != 0 { - return syscall.Errno(res) - } - return nil -} // Links come in the format of // name:alias @@ -386,14 +355,3 @@ func (c *checker) Exists(name string) bool { func generateRandomName(runtime *Runtime) (string, error) { return namesgenerator.GenerateRandomName(&checker{runtime}) } - -func CopyFile(dstFile, srcFile *os.File) error { - err := BtrfsReflink(dstFile.Fd(), srcFile.Fd()) - if err == nil { - return nil - } - - // Fall back to normal copy - _, err = io.Copy(dstFile, srcFile) - return err -} From c226ab6d9edb6b74547348c309c633f207bd4aa4 Mon Sep 17 00:00:00 2001 From: Ulysse Carion Date: Wed, 23 Oct 2013 09:19:15 -0700 Subject: [PATCH 046/162] Document setting up Vagrant-docker with the remote API --- contrib/vagrant-docker/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/contrib/vagrant-docker/README.md b/contrib/vagrant-docker/README.md index fa28e448eb..836f503059 100644 --- a/contrib/vagrant-docker/README.md +++ b/contrib/vagrant-docker/README.md @@ -17,3 +17,34 @@ meaning you can use Vagrant to control Docker containers. * [docker-provider](https://github.com/fgrehm/docker-provider) * [vagrant-shell](https://github.com/destructuring/vagrant-shell) + +## Setting up Vagrant-docker with the Remote API + +The initial Docker upstart script will not work because it runs on `127.0.0.1`, which is not accessible to the host machine. Instead, we need to change the script to connect to `0.0.0.0`. To do this, modify `/etc/init/docker.conf` to look like this: + +``` +description "Docker daemon" + +start on filesystem and started lxc-net +stop on runlevel [!2345] + +respawn + +script + /usr/bin/docker -d -H=tcp://0.0.0.0:4243/ +end script +``` + +Once that's done, you need to set up a SSH tunnel between your host machine and the vagrant machine that's running Docker. This can be done by running the following command in a host terminal: + +``` +ssh -L 4243:localhost:4243 -p 2222 vagrant@localhost +``` + +(The first 4243 is what your host can connect to, the second 4243 is what port Docker is running on in the vagrant machine, and the 2222 is the port Vagrant is providing for SSH. If VirtualBox is the VM you're using, you can see what value "2222" should be by going to: Network > Adapter 1 > Advanced > Port Forwarding in the VirtualBox GUI.) + +Note that because the port has been changed, to run docker commands from within the command line you must run them like this: + +``` +sudo docker -H 0.0.0.0:4243 < commands for docker > +``` From f1e44e0b0c6eff011c596008c75f7fdff09dbf5e Mon Sep 17 00:00:00 2001 From: John Warwick Date: Thu, 28 Nov 2013 09:55:15 -0500 Subject: [PATCH 047/162] Remove explanation of removed argument --- docs/sources/examples/hello_world.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/sources/examples/hello_world.rst b/docs/sources/examples/hello_world.rst index 99eaa2c483..c25070fcd8 100644 --- a/docs/sources/examples/hello_world.rst +++ b/docs/sources/examples/hello_world.rst @@ -131,8 +131,6 @@ Attach to the container to see the results in real-time. - **"docker attach**" This will allow us to attach to a background process to see what is going on. -- **"-sig-proxy=true"** Proxify all received signal to the process - (even in non-tty mode) - **$CONTAINER_ID** The Id of the container we want to attach too. Exit from the container attachment by pressing Control-C. From bdfe8ed403591022ac46371798783c7bbe6134c9 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Wed, 27 Nov 2013 23:14:34 -0500 Subject: [PATCH 048/162] Remove incorrect SIGKILL handler. As per POSIX signal handling SIGKILL does not work. Fixes #2928 --- server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index 3641e2fdc8..476890ee06 100644 --- a/server.go +++ b/server.go @@ -38,7 +38,7 @@ func init() { // jobInitApi runs the remote api server `srv` as a daemon, // Only one api server can run at the same time - this is enforced by a pidfile. -// The signals SIGINT, SIGKILL and SIGTERM are intercepted for cleanup. +// The signals SIGINT and SIGTERM are intercepted for cleanup. func jobInitApi(job *engine.Job) string { job.Logf("Creating server") srv, err := NewServer(job.Eng, ConfigFromJob(job)) @@ -53,7 +53,7 @@ func jobInitApi(job *engine.Job) string { } job.Logf("Setting up signal traps") c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) + signal.Notify(c, os.Interrupt, os.Signal(syscall.SIGTERM)) go func() { sig := <-c log.Printf("Received signal '%v', exiting\n", sig) From d47507791e14908e78cf38d415a9863c9ef75c5e Mon Sep 17 00:00:00 2001 From: Danny Yates Date: Thu, 28 Nov 2013 16:42:29 +0000 Subject: [PATCH 049/162] Stop invalid calls to Registry This code was resulting in a call for /v1/images///ancestry which the Registry doesn't understand. Furthermore, it was masking the original error. --- server.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/server.go b/server.go index 3641e2fdc8..b4635c8444 100644 --- a/server.go +++ b/server.go @@ -1011,16 +1011,9 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut localName = remoteName } - err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel) - if err == registry.ErrLoginRequired { + if err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel); err != nil { return err } - if err != nil { - if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { - return err - } - return nil - } return nil } From 438607ecc321a0256b7597cf278d4356632aac2f Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Nov 2013 16:49:40 -0700 Subject: [PATCH 050/162] Add proper dockerinit path support for distros that use FHS 2.3 --- utils/utils.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index cfdc73bb2e..87a99cd7a2 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -279,9 +279,16 @@ func DockerInitPath(localCopy string) string { var possibleInits = []string{ localCopy, filepath.Join(filepath.Dir(selfPath), "dockerinit"), - // "/usr/libexec includes internal binaries that are not intended to be executed directly by users or shell scripts. Applications may use a single subdirectory under /usr/libexec." + + // FHS 3.0 Draft: "/usr/libexec includes internal binaries that are not intended to be executed directly by users or shell scripts. Applications may use a single subdirectory under /usr/libexec." + // http://www.linuxbase.org/betaspecs/fhs/fhs.html#usrlibexec "/usr/libexec/docker/dockerinit", "/usr/local/libexec/docker/dockerinit", + + // FHS 2.3: "/usr/lib includes object files, libraries, and internal binaries that are not intended to be executed directly by users or shell scripts." + // http://refspecs.linuxfoundation.org/FHS_2.3/fhs-2.3.html#USRLIBLIBRARIESFORPROGRAMMINGANDPA + "/usr/lib/docker/dockerinit", + "/usr/local/lib/docker/dockerinit", } for _, dockerInit := range possibleInits { path, err := exec.LookPath(dockerInit) From 533067bba47acec0f65dc7e499944b9402bafb04 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 10:37:03 -0800 Subject: [PATCH 051/162] Rename file for consistency --- graphdriver/devmapper/{attachLoopback.go => attach_loopback.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename graphdriver/devmapper/{attachLoopback.go => attach_loopback.go} (100%) diff --git a/graphdriver/devmapper/attachLoopback.go b/graphdriver/devmapper/attach_loopback.go similarity index 100% rename from graphdriver/devmapper/attachLoopback.go rename to graphdriver/devmapper/attach_loopback.go From 9f46779d42c9b90a70c3c434d03a4502070e1b6d Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 19 Nov 2013 14:25:17 -0500 Subject: [PATCH 052/162] Wire in pprof handlers. Based on http://stackoverflow.com/questions/19591065/profiling-go-web-application-built-with-gorillas-mux-with-net-http-pprof --- api.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 2880d0e8bc..aadd79e3c8 100644 --- a/api.go +++ b/api.go @@ -15,6 +15,7 @@ import ( "mime" "net" "net/http" + "net/http/pprof" "os" "os/exec" "regexp" @@ -1037,9 +1038,21 @@ func makeHttpHandler(srv *Server, logging bool, localMethod string, localRoute s } } +func AttachProfiler(router *mux.Router) { + router.HandleFunc("/debug/pprof/", pprof.Index) + router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + router.HandleFunc("/debug/pprof/profile", pprof.Profile) + router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + router.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) + router.HandleFunc("/debug/pprof/goroutine", pprof.Handler("goroutine").ServeHTTP) + router.HandleFunc("/debug/pprof/threadcreate", pprof.Handler("threadcreate").ServeHTTP) +} + func createRouter(srv *Server, logging bool) (*mux.Router, error) { r := mux.NewRouter() - + if os.Getenv("DEBUG") != "" { + AttachProfiler(r) + } m := map[string]map[string]HttpApiFunc{ "GET": { "/events": getEvents, From a990b3aeb9640d35e73f99030e03e58e71924ed5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 11:02:53 -0800 Subject: [PATCH 053/162] Correct comments --- graphdriver/devmapper/attach_loopback.go | 4 ++-- graphdriver/devmapper/sys.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/graphdriver/devmapper/attach_loopback.go b/graphdriver/devmapper/attach_loopback.go index 0d556d81c2..f91bd20165 100644 --- a/graphdriver/devmapper/attach_loopback.go +++ b/graphdriver/devmapper/attach_loopback.go @@ -44,7 +44,7 @@ func openNextAvailableLoopback(index int, sparseFile *osFile) (loopFile *osFile, continue } - // Open the targeted loopback (use OpenFile because Open sets O_CLOEXEC) + // OpenFile adds O_CLOEXEC loopFile, err = osOpenFile(target, osORdWr, 0644) if err != nil { utils.Errorf("Error openning loopback device: %s", err) @@ -89,7 +89,7 @@ func attachLoopDevice(sparseName string) (loop *osFile, err error) { utils.Debugf("Error retrieving the next available loopback: %s", err) } - // Open the given sparse file (use OpenFile because Open sets O_CLOEXEC) + // OpenFile adds O_CLOEXEC sparseFile, err := osOpenFile(sparseName, osORdWr, 0644) if err != nil { utils.Errorf("Error openning sparse file %s: %s", sparseName, err) diff --git a/graphdriver/devmapper/sys.go b/graphdriver/devmapper/sys.go index 9cf124a9a5..e06165c824 100644 --- a/graphdriver/devmapper/sys.go +++ b/graphdriver/devmapper/sys.go @@ -34,9 +34,7 @@ var ( osRename = os.Rename osReadlink = os.Readlink - execRun = func(name string, args ...string) error { - return exec.Command(name, args...).Run() - } + execRun = func(name string, args ...string) error { return exec.Command(name, args...).Run() } ) const ( From 261bd0d187cbc8f650d77e5c93872332c290b164 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 11:53:09 -0800 Subject: [PATCH 054/162] Improve devmapper unit tests with syscall/ioctl --- graphdriver/devmapper/devmapper_wrapper.go | 7 +- graphdriver/devmapper/driver_test.go | 124 +++++++++++---------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/graphdriver/devmapper/devmapper_wrapper.go b/graphdriver/devmapper/devmapper_wrapper.go index 1b636d392f..3c195f72ea 100644 --- a/graphdriver/devmapper/devmapper_wrapper.go +++ b/graphdriver/devmapper/devmapper_wrapper.go @@ -54,21 +54,24 @@ type ( ) // FIXME: Make sure the values are defined in C +// IOCTL consts const ( + BlkGetSize64 = C.BLKGETSIZE64 + LoopSetFd = C.LOOP_SET_FD LoopCtlGetFree = C.LOOP_CTL_GET_FREE LoopGetStatus64 = C.LOOP_GET_STATUS64 LoopSetStatus64 = C.LOOP_SET_STATUS64 LoopClrFd = C.LOOP_CLR_FD LoopSetCapacity = C.LOOP_SET_CAPACITY +) +const ( LoFlagsAutoClear = C.LO_FLAGS_AUTOCLEAR LoFlagsReadOnly = C.LO_FLAGS_READ_ONLY LoFlagsPartScan = C.LO_FLAGS_PARTSCAN LoKeySize = C.LO_KEY_SIZE LoNameSize = C.LO_NAME_SIZE - - BlkGetSize64 = C.BLKGETSIZE64 ) var ( diff --git a/graphdriver/devmapper/driver_test.go b/graphdriver/devmapper/driver_test.go index 3204575dd9..3950152fb7 100644 --- a/graphdriver/devmapper/driver_test.go +++ b/graphdriver/devmapper/driver_test.go @@ -55,12 +55,6 @@ func denyAllDevmapper() { DmGetNextTarget = func(task *CDmTask, next uintptr, start, length *uint64, target, params *string) uintptr { panic("DmGetNextTarget: this method should not be called here") } - DmAttachLoopDevice = func(filename string, fd *int) string { - panic("DmAttachLoopDevice: this method should not be called here") - } - DmGetBlockSize = func(fd uintptr) (int64, sysErrno) { - panic("DmGetBlockSize: this method should not be called here") - } DmUdevWait = func(cookie uint) int { panic("DmUdevWait: this method should not be called here") } @@ -76,9 +70,6 @@ func denyAllDevmapper() { DmTaskDestroy = func(task *CDmTask) { panic("DmTaskDestroy: this method should not be called here") } - GetBlockSize = func(fd uintptr, size *uint64) sysErrno { - panic("GetBlockSize: this method should not be called here") - } LogWithErrnoInit = func() { panic("LogWithErrnoInit: this method should not be called here") } @@ -155,11 +146,10 @@ func (r Set) Assert(t *testing.T, names ...string) { func TestInit(t *testing.T) { var ( - calls = make(Set) - devicesAttached = make(Set) - taskMessages = make(Set) - taskTypes = make(Set) - home = mkTestDirectory(t) + calls = make(Set) + taskMessages = make(Set) + taskTypes = make(Set) + home = mkTestDirectory(t) ) defer osRemoveAll(home) @@ -233,29 +223,6 @@ func TestInit(t *testing.T) { taskMessages[message] = true return 1 } - var ( - fakeDataLoop = "/dev/loop42" - fakeMetadataLoop = "/dev/loop43" - fakeDataLoopFd = 42 - fakeMetadataLoopFd = 43 - ) - var attachCount int - DmAttachLoopDevice = func(filename string, fd *int) string { - calls["DmAttachLoopDevice"] = true - if _, exists := devicesAttached[filename]; exists { - t.Fatalf("Already attached %s", filename) - } - devicesAttached[filename] = true - // This will crash if fd is not dereferenceable - if attachCount == 0 { - attachCount++ - *fd = fakeDataLoopFd - return fakeDataLoop - } else { - *fd = fakeMetadataLoopFd - return fakeMetadataLoop - } - } DmTaskDestroy = func(task *CDmTask) { calls["DmTaskDestroy"] = true expectedTask := &task1 @@ -263,14 +230,6 @@ func TestInit(t *testing.T) { t.Fatalf("Wrong libdevmapper call\nExpected: DmTaskDestroy(%v)\nReceived: DmTaskDestroy(%v)\n", expectedTask, task) } } - fakeBlockSize := int64(4242 * 512) - DmGetBlockSize = func(fd uintptr) (int64, sysErrno) { - calls["DmGetBlockSize"] = true - if expectedFd := uintptr(42); fd != expectedFd { - t.Fatalf("Wrong libdevmapper call\nExpected: DmGetBlockSize(%v)\nReceived: DmGetBlockSize(%v)\n", expectedFd, fd) - } - return fakeBlockSize, 0 - } DmTaskAddTarget = func(task *CDmTask, start, size uint64, ttype, params string) int { calls["DmTaskSetTarget"] = true expectedTask := &task1 @@ -345,11 +304,9 @@ func TestInit(t *testing.T) { "DmTaskSetName", "DmTaskRun", "DmTaskGetInfo", - "DmAttachLoopDevice", "DmTaskDestroy", "execRun", "DmTaskCreate", - "DmGetBlockSize", "DmTaskSetTarget", "DmTaskSetCookie", "DmUdevWait", @@ -357,7 +314,6 @@ func TestInit(t *testing.T) { "DmTaskSetMessage", "DmTaskSetAddNode", ) - devicesAttached.Assert(t, path.Join(home, "devicemapper", "data"), path.Join(home, "devicemapper", "metadata")) taskTypes.Assert(t, "0", "6", "17") taskMessages.Assert(t, "create_thin 0", "set_transaction_id 0 1") } @@ -408,17 +364,9 @@ func mockAllDevmapper(calls Set) { calls["DmTaskSetMessage"] = true return 1 } - DmAttachLoopDevice = func(filename string, fd *int) string { - calls["DmAttachLoopDevice"] = true - return "/dev/loop42" - } DmTaskDestroy = func(task *CDmTask) { calls["DmTaskDestroy"] = true } - DmGetBlockSize = func(fd uintptr) (int64, sysErrno) { - calls["DmGetBlockSize"] = true - return int64(4242 * 512), 0 - } DmTaskAddTarget = func(task *CDmTask, start, size uint64, ttype, params string) int { calls["DmTaskSetTarget"] = true return 1 @@ -489,6 +437,32 @@ func TestDriverCreate(t *testing.T) { return false, nil } + sysSyscall = func(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) { + calls["sysSyscall"] = true + if trap != sysSysIoctl { + t.Fatalf("Unexpected syscall. Expecting SYS_IOCTL, received: %d", trap) + } + switch a2 { + case LoopSetFd: + calls["ioctl.loopsetfd"] = true + case LoopCtlGetFree: + calls["ioctl.loopctlgetfree"] = true + case LoopGetStatus64: + calls["ioctl.loopgetstatus"] = true + case LoopSetStatus64: + calls["ioctl.loopsetstatus"] = true + case LoopClrFd: + calls["ioctl.loopclrfd"] = true + case LoopSetCapacity: + calls["ioctl.loopsetcapacity"] = true + case BlkGetSize64: + calls["ioctl.blkgetsize"] = true + default: + t.Fatalf("Unexpected IOCTL. Received %d", a2) + } + return 0, 0, 0 + } + func() { d := newDriver(t) @@ -498,16 +472,18 @@ func TestDriverCreate(t *testing.T) { "DmTaskSetName", "DmTaskRun", "DmTaskGetInfo", - "DmAttachLoopDevice", "execRun", "DmTaskCreate", - "DmGetBlockSize", "DmTaskSetTarget", "DmTaskSetCookie", "DmUdevWait", "DmTaskSetSector", "DmTaskSetMessage", "DmTaskSetAddNode", + "sysSyscall", + "ioctl.blkgetsize", + "ioctl.loopsetfd", + "ioctl.loopsetstatus", ) if err := d.Create("1", ""); err != nil { @@ -579,6 +555,32 @@ func TestDriverRemove(t *testing.T) { return false, nil } + sysSyscall = func(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) { + calls["sysSyscall"] = true + if trap != sysSysIoctl { + t.Fatalf("Unexpected syscall. Expecting SYS_IOCTL, received: %d", trap) + } + switch a2 { + case LoopSetFd: + calls["ioctl.loopsetfd"] = true + case LoopCtlGetFree: + calls["ioctl.loopctlgetfree"] = true + case LoopGetStatus64: + calls["ioctl.loopgetstatus"] = true + case LoopSetStatus64: + calls["ioctl.loopsetstatus"] = true + case LoopClrFd: + calls["ioctl.loopclrfd"] = true + case LoopSetCapacity: + calls["ioctl.loopsetcapacity"] = true + case BlkGetSize64: + calls["ioctl.blkgetsize"] = true + default: + t.Fatalf("Unexpected IOCTL. Received %d", a2) + } + return 0, 0, 0 + } + func() { d := newDriver(t) @@ -588,16 +590,18 @@ func TestDriverRemove(t *testing.T) { "DmTaskSetName", "DmTaskRun", "DmTaskGetInfo", - "DmAttachLoopDevice", "execRun", "DmTaskCreate", - "DmGetBlockSize", "DmTaskSetTarget", "DmTaskSetCookie", "DmUdevWait", "DmTaskSetSector", "DmTaskSetMessage", "DmTaskSetAddNode", + "sysSyscall", + "ioctl.blkgetsize", + "ioctl.loopsetfd", + "ioctl.loopsetstatus", ) if err := d.Create("1", ""); err != nil { From 597e0e69b4c8521f39691d0a07d1f31b7116a337 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 28 Nov 2013 12:16:57 -0800 Subject: [PATCH 055/162] split in 3 files --- commands.go | 2 +- graph.go | 2 +- server.go | 34 +++---- utils/jsonmessage.go | 118 ++++++++++++++++++++++++ utils/progressreader.go | 55 +++++++++++ utils/streamformatter.go | 68 ++++++++++++++ utils/utils.go | 192 --------------------------------------- 7 files changed, 260 insertions(+), 211 deletions(-) create mode 100644 utils/jsonmessage.go create mode 100644 utils/progressreader.go create mode 100644 utils/streamformatter.go diff --git a/commands.go b/commands.go index d992db2e6c..f2e8d0de2a 100644 --- a/commands.go +++ b/commands.go @@ -202,7 +202,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoying to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf, true) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf, true, "", "Uploading context") } // Upload the build context v := &url.Values{} diff --git a/graph.go b/graph.go index 04d77b9146..19c668830e 100644 --- a/graph.go +++ b/graph.go @@ -205,7 +205,7 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, if err != nil { return nil, err } - return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp) + return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf, true, "", "Buffering to disk"), tmp) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/server.go b/server.go index 3783c7bded..bd4dc22740 100644 --- a/server.go +++ b/server.go @@ -451,7 +451,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf, false), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf, false, "", "Downloading"), path); err != nil { return err } // FIXME: Handle custom repo, tag comment, author @@ -761,7 +761,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin if err != nil { return err } - out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependent layers")) + out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil)) // FIXME: Try to stream the images? // FIXME: Launch the getRemoteImage() in goroutines @@ -776,33 +776,33 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin defer srv.poolRemove("pull", "layer:"+id) if !srv.runtime.graph.Exists(id) { - out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil)) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { - out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) // FIXME: Keep going in case of error? return err } img, err := NewImgJSON(imgJSON) if err != nil { - out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) return fmt.Errorf("Failed to parse json: %s", err) } // Get the layer - out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling fs layer", nil)) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { - out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil { - out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependent layers")) + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"), img); err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil)) return err } } - out.Write(sf.FormatProgress(utils.TruncateID(id), "Download", "complete")) + out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil)) } return nil @@ -875,29 +875,29 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName } defer srv.poolRemove("pull", "img:"+img.ID) - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s", img.Tag, localName))) + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, localName), nil)) success := false var lastErr error for _, ep := range repoData.Endpoints { - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s", img.Tag, localName, ep))) + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, localName, ep), nil)) if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { // Its not ideal that only the last error is returned, it would be better to concatenate the errors. // As the error is also given to the output stream the user will see the error. lastErr = err - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err))) + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, localName, ep, err), nil)) continue } success = true break } if !success { - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Error pulling", fmt.Sprintf("image (%s) from %s, %s", img.Tag, localName, lastErr))) + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, %s", img.Tag, localName, lastErr), nil)) if parallel { errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") return } } - out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download", "complete")) + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) if parallel { errors <- nil @@ -1171,7 +1171,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, defer os.RemoveAll(layerData.Name()) // Send the layer - checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, false), ep, token, jsonRaw) + checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf, false, "", "Pushing"), ep, token, jsonRaw) if err != nil { return "", err } @@ -1251,7 +1251,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf, true) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf, true, "", "Importing") } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go new file mode 100644 index 0000000000..93c40116e4 --- /dev/null +++ b/utils/jsonmessage.go @@ -0,0 +1,118 @@ +package utils + +import ( + "encoding/json" + "fmt" + "io" + "time" +) + +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message +} + +type JSONProgress struct { + Current int `json:"current,omitempty"` + Total int `json:"total,omitempty"` +} + +func (p *JSONProgress) String() string { + if p.Current == 0 && p.Total == 0 { + return "" + } + current := HumanSize(int64(p.Current)) + if p.Total == 0 { + return fmt.Sprintf("%8v/?", current) + } + total := HumanSize(int64(p.Total)) + percentage := float64(p.Current) / float64(p.Total) * 100 + return fmt.Sprintf("%8v/%v (%.0f%%)", current, total, percentage) +} + +type JSONMessage struct { + Status string `json:"status,omitempty"` + Progress *JSONProgress `json:"progressDetail,omitempty"` + ProgressMessage string `json:"progress,omitempty"` //deprecated + ID string `json:"id,omitempty"` + From string `json:"from,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated +} + +func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { + if jm.Error != nil { + if jm.Error.Code == 401 { + return fmt.Errorf("Authentication is required.") + } + return jm.Error + } + endl := "" + if isTerminal { + // [2K = erase entire current line + fmt.Fprintf(out, "%c[2K\r", 27) + endl = "\r" + } + if jm.Time != 0 { + fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) + } + if jm.ID != "" { + fmt.Fprintf(out, "%s: ", jm.ID) + } + if jm.From != "" { + fmt.Fprintf(out, "(from %s) ", jm.From) + } + if jm.Progress != nil { + fmt.Fprintf(out, "%s %s%s", jm.Status, jm.Progress.String(), endl) + } else if jm.ProgressMessage != "" { //deprecated + fmt.Fprintf(out, "%s %s%s", jm.Status, jm.ProgressMessage, endl) + } else { + fmt.Fprintf(out, "%s%s\n", jm.Status, endl) + } + return nil +} + +func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) error { + dec := json.NewDecoder(in) + ids := make(map[string]int) + diff := 0 + for { + jm := JSONMessage{} + if err := dec.Decode(&jm); err == io.EOF { + break + } else if err != nil { + return err + } + if (jm.Progress != nil || jm.ProgressMessage != "") && jm.ID != "" { + line, ok := ids[jm.ID] + if !ok { + line = len(ids) + ids[jm.ID] = line + fmt.Fprintf(out, "\n") + diff = 0 + } else { + diff = len(ids) - line + } + if isTerminal { + // [{diff}A = move cursor up diff rows + fmt.Fprintf(out, "%c[%dA", 27, diff) + } + } + err := jm.Display(out, isTerminal) + if jm.ID != "" { + if isTerminal { + // [{diff}B = move cursor down diff rows + fmt.Fprintf(out, "%c[%dB", 27, diff) + } + } + if err != nil { + return err + } + } + return nil +} diff --git a/utils/progressreader.go b/utils/progressreader.go new file mode 100644 index 0000000000..db332a6d1c --- /dev/null +++ b/utils/progressreader.go @@ -0,0 +1,55 @@ +package utils + +import ( + "io" +) + +// Reader with progress bar +type progressReader struct { + reader io.ReadCloser // Stream to read from + output io.Writer // Where to send progress bar to + progress JSONProgress + // readTotal int // Expected stream length (bytes) + // readProgress int // How much has been read so far (bytes) + lastUpdate int // How many bytes read at least update + ID string + action string + // template string // Template to print. Default "%v/%v (%v)" + sf *StreamFormatter + newLine bool +} + +func (r *progressReader) Read(p []byte) (n int, err error) { + read, err := io.ReadCloser(r.reader).Read(p) + r.progress.Current += read + updateEvery := 1024 * 512 //512kB + if r.progress.Total > 0 { + // Update progress for every 1% read if 1% < 512kB + if increment := int(0.01 * float64(r.progress.Total)); increment < updateEvery { + updateEvery = increment + } + } + if r.progress.Current-r.lastUpdate > updateEvery || err != nil { + r.output.Write(r.sf.FormatProgress(r.ID, r.action, &r.progress)) + r.lastUpdate = r.progress.Current + } + // Send newline when complete + if r.newLine && err != nil { + r.output.Write(r.sf.FormatStatus("", "")) + } + return read, err +} +func (r *progressReader) Close() error { + return io.ReadCloser(r.reader).Close() +} +func ProgressReader(r io.ReadCloser, size int, output io.Writer, sf *StreamFormatter, newline bool, ID, action string) *progressReader { + return &progressReader{ + reader: r, + output: NewWriteFlusher(output), + ID: ID, + action: action, + progress: JSONProgress{Total: size}, + sf: sf, + newLine: newline, + } +} diff --git a/utils/streamformatter.go b/utils/streamformatter.go new file mode 100644 index 0000000000..a28cd5050e --- /dev/null +++ b/utils/streamformatter.go @@ -0,0 +1,68 @@ +package utils + +import ( + "encoding/json" + "fmt" +) + +type StreamFormatter struct { + json bool + used bool +} + +func NewStreamFormatter(json bool) *StreamFormatter { + return &StreamFormatter{json, false} +} + +func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { + sf.used = true + str := fmt.Sprintf(format, a...) + if sf.json { + b, err := json.Marshal(&JSONMessage{ID: id, Status: str}) + if err != nil { + return sf.FormatError(err) + } + return b + } + return []byte(str + "\r\n") +} + +func (sf *StreamFormatter) FormatError(err error) []byte { + sf.used = true + if sf.json { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { + return b + } + return []byte("{\"error\":\"format error\"}") + } + return []byte("Error: " + err.Error() + "\r\n") +} + +func (sf *StreamFormatter) FormatProgress(id, action string, progress *JSONProgress) []byte { + if progress == nil { + progress = &JSONProgress{} + } + sf.used = true + if sf.json { + + b, err := json.Marshal(&JSONMessage{ + Status: action, + ProgressMessage: progress.String(), + Progress: progress, + ID: id, + }) + if err != nil { + return nil + } + return b + } + return []byte(action + " " + progress.String() + "\r") +} + +func (sf *StreamFormatter) Used() bool { + return sf.used +} diff --git a/utils/utils.go b/utils/utils.go index cfdc73bb2e..0ab2245940 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -94,56 +94,6 @@ func Errorf(format string, a ...interface{}) { logf("error", format, a...) } -// Reader with progress bar -type progressReader struct { - reader io.ReadCloser // Stream to read from - output io.Writer // Where to send progress bar to - readTotal int // Expected stream length (bytes) - readProgress int // How much has been read so far (bytes) - lastUpdate int // How many bytes read at least update - template string // Template to print. Default "%v/%v (%v)" - sf *StreamFormatter - newLine bool -} - -func (r *progressReader) Read(p []byte) (n int, err error) { - read, err := io.ReadCloser(r.reader).Read(p) - r.readProgress += read - updateEvery := 1024 * 512 //512kB - if r.readTotal > 0 { - // Update progress for every 1% read if 1% < 512kB - if increment := int(0.01 * float64(r.readTotal)); increment < updateEvery { - updateEvery = increment - } - } - if r.readProgress-r.lastUpdate > updateEvery || err != nil { - if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) - } else { - fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a") - } - r.lastUpdate = r.readProgress - } - // Send newline when complete - if r.newLine && err != nil { - r.output.Write(r.sf.FormatStatus("", "")) - } - return read, err -} -func (r *progressReader) Close() error { - return io.ReadCloser(r.reader).Close() -} -func ProgressReader(r io.ReadCloser, size int, output io.Writer, tpl []byte, sf *StreamFormatter, newline bool) *progressReader { - return &progressReader{ - reader: r, - output: NewWriteFlusher(output), - readTotal: size, - template: string(tpl), - sf: sf, - newLine: newline, - } -} - // HumanDuration returns a human-readable approximation of a duration // (eg. "About a minute", "4 hours ago", etc.) func HumanDuration(d time.Duration) string { @@ -754,25 +704,6 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } -type JSONError struct { - Code int `json:"code,omitempty"` - Message string `json:"message,omitempty"` -} - -type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - ErrorMessage string `json:"error,omitempty"` //deprecated - ID string `json:"id,omitempty"` - From string `json:"from,omitempty"` - Time int64 `json:"time,omitempty"` - Error *JSONError `json:"errorDetail,omitempty"` -} - -func (e *JSONError) Error() string { - return e.Message -} - func NewHTTPRequestError(msg string, res *http.Response) error { return &JSONError{ Message: msg, @@ -780,129 +711,6 @@ func NewHTTPRequestError(msg string, res *http.Response) error { } } -func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { - if jm.Error != nil { - if jm.Error.Code == 401 { - return fmt.Errorf("Authentication is required.") - } - return jm.Error - } - endl := "" - if isTerminal { - // [2K = erase entire current line - fmt.Fprintf(out, "%c[2K\r", 27) - endl = "\r" - } - if jm.Time != 0 { - fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) - } - if jm.ID != "" { - fmt.Fprintf(out, "%s: ", jm.ID) - } - if jm.From != "" { - fmt.Fprintf(out, "(from %s) ", jm.From) - } - if jm.Progress != "" { - fmt.Fprintf(out, "%s %s%s", jm.Status, jm.Progress, endl) - } else { - fmt.Fprintf(out, "%s%s\n", jm.Status, endl) - } - return nil -} - -func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) error { - dec := json.NewDecoder(in) - ids := make(map[string]int) - diff := 0 - for { - jm := JSONMessage{} - if err := dec.Decode(&jm); err == io.EOF { - break - } else if err != nil { - return err - } - if jm.Progress != "" && jm.ID != "" { - line, ok := ids[jm.ID] - if !ok { - line = len(ids) - ids[jm.ID] = line - fmt.Fprintf(out, "\n") - diff = 0 - } else { - diff = len(ids) - line - } - if isTerminal { - // [{diff}A = move cursor up diff rows - fmt.Fprintf(out, "%c[%dA", 27, diff) - } - } - err := jm.Display(out, isTerminal) - if jm.ID != "" { - if isTerminal { - // [{diff}B = move cursor down diff rows - fmt.Fprintf(out, "%c[%dB", 27, diff) - } - } - if err != nil { - return err - } - } - return nil -} - -type StreamFormatter struct { - json bool - used bool -} - -func NewStreamFormatter(json bool) *StreamFormatter { - return &StreamFormatter{json, false} -} - -func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { - sf.used = true - str := fmt.Sprintf(format, a...) - if sf.json { - b, err := json.Marshal(&JSONMessage{ID: id, Status: str}) - if err != nil { - return sf.FormatError(err) - } - return b - } - return []byte(str + "\r\n") -} - -func (sf *StreamFormatter) FormatError(err error) []byte { - sf.used = true - if sf.json { - jsonError, ok := err.(*JSONError) - if !ok { - jsonError = &JSONError{Message: err.Error()} - } - if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { - return b - } - return []byte("{\"error\":\"format error\"}") - } - return []byte("Error: " + err.Error() + "\r\n") -} - -func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte { - sf.used = true - if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID: id}) - if err != nil { - return nil - } - return b - } - return []byte(action + " " + progress + "\r") -} - -func (sf *StreamFormatter) Used() bool { - return sf.used -} - func IsURL(str string) bool { return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") } From 2bbc90e92ff140c29891e92c9ce320ca7d19cd57 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 12:24:04 -0800 Subject: [PATCH 056/162] Make volumes opts more strict --- opts.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opts.go b/opts.go index 826091bea6..80785a5161 100644 --- a/opts.go +++ b/opts.go @@ -100,6 +100,10 @@ func ValidateLink(val string) (string, error) { func ValidatePath(val string) (string, error) { var containerPath string + if strings.Count(val, ":") > 2 { + return val, fmt.Errorf("bad format for volumes: %s", val) + } + splited := strings.SplitN(val, ":", 2) if len(splited) == 1 { containerPath = splited[0] From ebc36b879d4197a9455b325b529da212539e180c Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 28 Nov 2013 12:37:07 -0800 Subject: [PATCH 057/162] add progressbar and time --- utils/jsonmessage.go | 15 +++++++++++---- utils/progressreader.go | 3 ++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index 93c40116e4..a0aeba2e7a 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "time" ) @@ -17,8 +18,9 @@ func (e *JSONError) Error() string { } type JSONProgress struct { - Current int `json:"current,omitempty"` - Total int `json:"total,omitempty"` + Current int `json:"current,omitempty"` + Total int `json:"total,omitempty"` + Start int64 `json:"start,omitempty"` } func (p *JSONProgress) String() string { @@ -30,8 +32,13 @@ func (p *JSONProgress) String() string { return fmt.Sprintf("%8v/?", current) } total := HumanSize(int64(p.Total)) - percentage := float64(p.Current) / float64(p.Total) * 100 - return fmt.Sprintf("%8v/%v (%.0f%%)", current, total, percentage) + percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + + fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) + perEntry := fromStart / time.Duration(p.Current) + left := time.Duration(p.Total-p.Current) * perEntry + left = (left / time.Second) * time.Second + return fmt.Sprintf("[%s>%s] %8v/%v %s", strings.Repeat("=", percentage), strings.Repeat(" ", 50-percentage), current, total, left.String()) } type JSONMessage struct { diff --git a/utils/progressreader.go b/utils/progressreader.go index db332a6d1c..81f076682e 100644 --- a/utils/progressreader.go +++ b/utils/progressreader.go @@ -2,6 +2,7 @@ package utils import ( "io" + "time" ) // Reader with progress bar @@ -48,7 +49,7 @@ func ProgressReader(r io.ReadCloser, size int, output io.Writer, sf *StreamForma output: NewWriteFlusher(output), ID: ID, action: action, - progress: JSONProgress{Total: size}, + progress: JSONProgress{Total: size, Start: time.Now().UTC().Unix()}, sf: sf, newLine: newline, } From b36dd3f9ccc61e42024cf1f2799c73f0efc77c75 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 28 Nov 2013 14:40:17 -0800 Subject: [PATCH 058/162] fix display on test --- utils/streamformatter.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/streamformatter.go b/utils/streamformatter.go index a28cd5050e..60863d2aa6 100644 --- a/utils/streamformatter.go +++ b/utils/streamformatter.go @@ -60,7 +60,11 @@ func (sf *StreamFormatter) FormatProgress(id, action string, progress *JSONProgr } return b } - return []byte(action + " " + progress.String() + "\r") + endl := "\r" + if progress.String() == "" { + endl += "\n" + } + return []byte(action + " " + progress.String() + endl) } func (sf *StreamFormatter) Used() bool { From 1fe1b216ad9b60f6b25a5dadda5fe26c3602b054 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 27 Nov 2013 17:10:20 -0800 Subject: [PATCH 059/162] Return process exit code for attach Fixes #2240 --- commands.go | 9 +++++++++ docs/sources/commandline/cli.rst | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index d992db2e6c..d52a463638 100644 --- a/commands.go +++ b/commands.go @@ -1578,6 +1578,15 @@ func (cli *DockerCli) CmdAttach(args ...string) error { if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, in, cli.out, cli.err, nil); err != nil { return err } + + _, status, err := getExitCode(cli, cmd.Arg(0)) + if err != nil { + return err + } + if status != 0 { + return &utils.StatusError{Status: status} + } + return nil } diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 9635675a77..01f839736a 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -66,7 +66,8 @@ To run the daemon with debug output, use ``docker -d -D`` You can detach from the container again (and leave it running) with ``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of -the Docker client when it quits. +the Docker client when it quits. When you detach from the container's +process the exit code will be retuned to the client. To stop a container, use ``docker stop`` From ad43d88af5bda8dc5b3d06f64de380bb985191ba Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 16:12:45 -0800 Subject: [PATCH 060/162] Make race condition more obvious by performing more asserts --- integration/api_test.go | 6 +++--- integration/commands_test.go | 14 +++++++------- integration/container_test.go | 2 +- integration/server_test.go | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/integration/api_test.go b/integration/api_test.go index a66cbe561f..bd6280af8b 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -454,7 +454,7 @@ func TestGetContainersTop(t *testing.T) { // Make sure sh spawn up cat setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { in, out := containerAttach(eng, containerID, t) - if err := assertPipe("hello\n", "hello", out, in, 15); err != nil { + if err := assertPipe("hello\n", "hello", out, in, 150); err != nil { t.Fatal(err) } }) @@ -877,7 +877,7 @@ func TestPostContainersAttach(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string([]byte{1, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{1, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) @@ -956,7 +956,7 @@ func TestPostContainersAttachStderr(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string([]byte{2, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{2, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) diff --git a/integration/commands_test.go b/integration/commands_test.go index 37bedf7f0c..7daebf3cd2 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -213,7 +213,7 @@ func TestRunExit(t *testing.T) { }() setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) @@ -268,7 +268,7 @@ func TestRunDisconnect(t *testing.T) { }() setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) @@ -330,7 +330,7 @@ func TestRunDisconnectTty(t *testing.T) { container := globalRuntime.List()[0] setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) @@ -432,7 +432,7 @@ func TestRunDetach(t *testing.T) { }() setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) @@ -513,7 +513,7 @@ func TestAttachDetach(t *testing.T) { }() setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { if err != io.ErrClosedPipe { t.Fatal(err) } @@ -575,7 +575,7 @@ func TestAttachDetachTruncatedID(t *testing.T) { }() setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { if err != io.ErrClosedPipe { t.Fatal(err) } @@ -648,7 +648,7 @@ func TestAttachDisconnect(t *testing.T) { }() setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) diff --git a/integration/container_test.go b/integration/container_test.go index 93a00a7286..05eb48728c 100644 --- a/integration/container_test.go +++ b/integration/container_test.go @@ -462,7 +462,7 @@ func TestKillDifferentUser(t *testing.T) { setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { out, _ := container.StdoutPipe() in, _ := container.StdinPipe() - if err := assertPipe("hello\n", "hello", out, in, 15); err != nil { + if err := assertPipe("hello\n", "hello", out, in, 150); err != nil { t.Fatal(err) } }) diff --git a/integration/server_test.go b/integration/server_test.go index 494e23fef3..24e109ab76 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -183,11 +183,11 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerRestart(id, 15); err != nil { + if err := srv.ContainerRestart(id, 150); err != nil { t.Fatal(err) } - if err := srv.ContainerStop(id, 15); err != nil { + if err := srv.ContainerStop(id, 150); err != nil { t.Fatal(err) } From d3cc558d140283a4ce051827705814cf4ca96f7a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 28 Nov 2013 16:28:31 -0800 Subject: [PATCH 061/162] add test --- utils/jsonmessage_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 utils/jsonmessage_test.go diff --git a/utils/jsonmessage_test.go b/utils/jsonmessage_test.go new file mode 100644 index 0000000000..9421d97a86 --- /dev/null +++ b/utils/jsonmessage_test.go @@ -0,0 +1,24 @@ +package utils + +import ( + "testing" +) + +func TestError(t *testing.T) { + je := JSONError{404, "Not found"} + if je.Error() != "Not found" { + t.Fatalf("Expected 'Not found' got '%s'", je.Error()) + } +} + +func TestProgress(t *testing.T) { + jp := JSONProgress{0, 0, 0} + if jp.String() != "" { + t.Fatalf("Expected empty string, got '%s'", jp.String()) + } + + jp2 := JSONProgress{1, 0, 0} + if jp2.String() != " 1 B/?" { + t.Fatalf("Expected ' 1/?', got '%s'", jp2.String()) + } +} From 77c94175bdc387e7f43876f7097b970f67116054 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 28 Nov 2013 16:57:51 -0800 Subject: [PATCH 062/162] Make CopyEscapable consistent with Copy and return `nil` in case of success instead of io.EOF --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index cfdc73bb2e..f62aa12ff5 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -543,7 +543,7 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) if err := src.Close(); err != nil { return 0, err } - return 0, io.EOF + return 0, nil } } // ---- End of docker From 8291d509c234779dff081ab579b6ff30358e7501 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Thu, 28 Nov 2013 17:19:26 -0800 Subject: [PATCH 063/162] Fixed some grammar and one other line about needing the postgresql-client for connecting to the service. --- docs/sources/examples/postgresql_service.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/examples/postgresql_service.rst b/docs/sources/examples/postgresql_service.rst index 82ca8b59ca..2b57853362 100644 --- a/docs/sources/examples/postgresql_service.rst +++ b/docs/sources/examples/postgresql_service.rst @@ -96,7 +96,7 @@ uncomment ``listen_addresses`` so it is as follows: This PostgreSQL setup is for development only purposes. Refer to PostgreSQL documentation how to fine-tune these settings so that it - is enough secure. + is secure enough. Exit. @@ -121,9 +121,9 @@ Finally, run PostgreSQL server via ``docker``. -D /var/lib/postgresql/9.3/main \ -c config_file=/etc/postgresql/9.3/main/postgresql.conf') -Connect the PostgreSQL server using ``psql`` (You will need postgres installed +Connect the PostgreSQL server using ``psql`` (You will need the postgresql client installed on the machine. For ubuntu, use something like -``sudo apt-get install postgresql``). +``sudo apt-get install postgresql-client``). .. code-block:: bash From ba5268d382dec7b731717feec77b6e7d2e6cfdaf Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Sat, 16 Nov 2013 20:49:42 +1000 Subject: [PATCH 064/162] re-word the help for docker import to make it clear that this will be an empty image containing only the files in the tar file --- commands.go | 2 +- docs/sources/commandline/cli.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 5dd76fe187..2457685739 100644 --- a/commands.go +++ b/commands.go @@ -928,7 +928,7 @@ func (cli *DockerCli) CmdKill(args ...string) error { } func (cli *DockerCli) CmdImport(args ...string) error { - cmd := cli.Subcmd("import", "URL|- [REPOSITORY[:TAG]]", "Create a new filesystem image from the contents of a tarball(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).") + cmd := cli.Subcmd("import", "URL|- [REPOSITORY[:TAG]]", "Create an empty filesystem image and import the contents of the tarball (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) into it, then optionally tag it.") if err := cmd.Parse(args); err != nil { return nil diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index e35475ae96..a4eaa820be 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -557,7 +557,8 @@ Displaying image hierarchy Usage: docker import URL|- [REPOSITORY[:TAG]] - Create a new filesystem image from the contents of a tarball + Create an empty filesystem image and import the contents of the tarball + (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) into it, then optionally tag it. At this time, the URL must start with ``http`` and point to a single file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a From fe727e2a87fa086d728664c396fd44f4be6d6afd Mon Sep 17 00:00:00 2001 From: cressie176 Date: Fri, 29 Nov 2013 10:02:53 +0000 Subject: [PATCH 065/162] Closing connection after ping --- registry/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registry/registry.go b/registry/registry.go index f02e3cf477..d3d9f2be54 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -47,6 +47,8 @@ func pingRegistryEndpoint(endpoint string) error { if err != nil { return err } + defer resp.Body.Close() + if resp.Header.Get("X-Docker-Registry-Version") == "" { return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") } From e535f544c7bc9c32b23e5505110a5513ff36be5a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 07:39:51 -0800 Subject: [PATCH 066/162] Make sure the container is running before testing against it (TestAttachDetach) --- integration/commands_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/integration/commands_test.go b/integration/commands_test.go index 7daebf3cd2..8bc1c99ec7 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -481,6 +481,17 @@ func TestAttachDetach(t *testing.T) { var container *docker.Container + setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { + for { + l := globalRuntime.List() + if len(l) == 1 && l[0].State.IsRunning() { + container = l[0] + break + } + time.Sleep(10 * time.Millisecond) + } + }) + setTimeout(t, "Reading container's id timed out", 10*time.Second, func() { buf := make([]byte, 1024) n, err := stdout.Read(buf) @@ -488,8 +499,6 @@ func TestAttachDetach(t *testing.T) { t.Fatal(err) } - container = globalRuntime.List()[0] - if strings.Trim(string(buf[:n]), " \r\n") != container.ID { t.Fatalf("Wrong ID received. Expect %s, received %s", container.ID, buf[:n]) } From fbebe20bc69648c046e3818ca744ae246092a782 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 07:40:44 -0800 Subject: [PATCH 067/162] Add a GetPtyMaster() method to container to retrieve the pty from an other package. We could also have put ptyMaster public, but then we need to ignore it in json otherwise the Marshalling fails. I think it is cleaner that way. --- container.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/container.go b/container.go index 49cb33b536..3e5f8a8d74 100644 --- a/container.go +++ b/container.go @@ -24,6 +24,11 @@ import ( "time" ) +var ( + ErrNotATTY = errors.New("The PTY is not a file") + ErrNoTTY = errors.New("No PTY found") +) + type Container struct { sync.Mutex root string // Path to the "home" of the container, including metadata. @@ -1405,3 +1410,13 @@ func (container *Container) Exposes(p Port) bool { _, exists := container.Config.ExposedPorts[p] return exists } + +func (container *Container) GetPtyMaster() (*os.File, error) { + if container.ptyMaster == nil { + return nil, ErrNoTTY + } + if pty, ok := container.ptyMaster.(*os.File); ok { + return pty, nil + } + return nil, ErrNotATTY +} From 67e9e0e11bb932ef9113ac91b2c1b0af6ee4db6d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 07:42:26 -0800 Subject: [PATCH 068/162] Make the PTY in raw mode before assert test (TestAttachDetach) --- integration/commands_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/integration/commands_test.go b/integration/commands_test.go index 8bc1c99ec7..f2b14482b2 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/dotcloud/docker" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/term" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -507,6 +508,17 @@ func TestAttachDetach(t *testing.T) { <-ch }) + pty, err := container.GetPtyMaster() + if err != nil { + t.Fatal(err) + } + + state, err := term.MakeRaw(pty.Fd()) + if err != nil { + t.Fatal(err) + } + defer term.RestoreTerminal(pty.Fd(), state) + stdin, stdinPipe = io.Pipe() stdout, stdoutPipe = io.Pipe() cli = docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) From 63d6cbe3e4d6f29c2491b0f1f505ef79b7191d8e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 09:11:20 -0800 Subject: [PATCH 069/162] Actually test the detach (was not the case before) --- commands.go | 6 ++++++ integration/commands_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/commands.go b/commands.go index db752447f0..24218d284c 100644 --- a/commands.go +++ b/commands.go @@ -2394,6 +2394,12 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea if stdout != nil { receiveStdout = utils.Go(func() (err error) { + defer func() { + if in != nil { + in.Close() + } + }() + // When TTY is ON, use regular copy if setRawTerminal { _, err = io.Copy(stdout, br) diff --git a/integration/commands_test.go b/integration/commands_test.go index f2b14482b2..75d09facb4 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -542,18 +542,18 @@ func TestAttachDetach(t *testing.T) { }) setTimeout(t, "Escape sequence timeout", 5*time.Second, func() { - stdinPipe.Write([]byte{16, 17}) - if err := stdinPipe.Close(); err != nil { - t.Fatal(err) - } + stdinPipe.Write([]byte{16}) + time.Sleep(100 * time.Millisecond) + stdinPipe.Write([]byte{17}) }) - closeWrap(stdin, stdinPipe, stdout, stdoutPipe) // wait for CmdRun to return setTimeout(t, "Waiting for CmdAttach timed out", 15*time.Second, func() { <-ch }) + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) + time.Sleep(500 * time.Millisecond) if !container.State.IsRunning() { t.Fatal("The detached container should be still running") From aa68656cd3aefe2a64dc259e8d19c72010e4f85b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 09:52:44 -0800 Subject: [PATCH 070/162] Fix term.RestoreTerminal behavior --- term/term.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/term/term.go b/term/term.go index 8c53a20ca6..50425a8602 100644 --- a/term/term.go +++ b/term/term.go @@ -1,12 +1,17 @@ package term import ( + "errors" "os" "os/signal" "syscall" "unsafe" ) +var ( + ErrInvalidState = errors.New("Invlide terminal state") +) + type State struct { termios Termios } @@ -47,8 +52,14 @@ func IsTerminal(fd uintptr) bool { // Restore restores the terminal connected to the given file descriptor to a // previous state. func RestoreTerminal(fd uintptr, state *State) error { + if state == nil { + return ErrInvalidState + } _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios))) - return err + if err != 0 { + return err + } + return nil } func SaveState(fd uintptr) (*State, error) { From c13821ad0bfa596a8bbc06a494500ee39a35e429 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 09:55:15 -0800 Subject: [PATCH 071/162] Make sure the termcaps are restored after hijack --- commands.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/commands.go b/commands.go index 24218d284c..dbf295cf13 100644 --- a/commands.go +++ b/commands.go @@ -2392,10 +2392,23 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea var receiveStdout chan error + var oldState *term.State + + if in != nil && setRawTerminal && cli.isTerminal && os.Getenv("NORAW") == "" { + oldState, err = term.SetRawTerminal(cli.terminalFd) + if err != nil { + return err + } + defer term.RestoreTerminal(cli.terminalFd, oldState) + } + if stdout != nil { receiveStdout = utils.Go(func() (err error) { defer func() { if in != nil { + if setRawTerminal && cli.isTerminal { + term.RestoreTerminal(cli.terminalFd, oldState) + } in.Close() } }() @@ -2411,14 +2424,6 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea }) } - if in != nil && setRawTerminal && cli.isTerminal && os.Getenv("NORAW") == "" { - oldState, err := term.SetRawTerminal(cli.terminalFd) - if err != nil { - return err - } - defer term.RestoreTerminal(cli.terminalFd, oldState) - } - sendStdin := utils.Go(func() error { if in != nil { io.Copy(rwc, in) From 697be6aaa009cd2bea5f07ae0b0780703e6565e1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 09:57:59 -0800 Subject: [PATCH 072/162] Create helper function for tests --- integration/commands_test.go | 66 ++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/integration/commands_test.go b/integration/commands_test.go index 75d09facb4..fa8f25836c 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -32,6 +32,47 @@ func closeWrap(args ...io.Closer) error { return nil } +func setRaw(t *testing.T, c *docker.Container) *term.State { + pty, err := c.GetPtyMaster() + if err != nil { + t.Fatal(err) + } + state, err := term.MakeRaw(pty.Fd()) + if err != nil { + t.Fatal(err) + } + return state +} + +func unsetRaw(t *testing.T, c *docker.Container, state *term.State) { + pty, err := c.GetPtyMaster() + if err != nil { + t.Fatal(err) + } + term.RestoreTerminal(pty.Fd(), state) +} + +func waitContainerStart(t *testing.T, timeout time.Duration) *docker.Container { + var container *docker.Container + + setTimeout(t, "Waiting for the container to be started timed out", timeout, func() { + for { + l := globalRuntime.List() + if len(l) == 1 && l[0].State.IsRunning() { + container = l[0] + break + } + time.Sleep(10 * time.Millisecond) + } + }) + + if container == nil { + t.Fatal("An error occured while waiting for the container to start") + } + + return container +} + func setTimeout(t *testing.T, msg string, d time.Duration, f func()) { c := make(chan bool) @@ -480,18 +521,7 @@ func TestAttachDetach(t *testing.T) { } }() - var container *docker.Container - - setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { - for { - l := globalRuntime.List() - if len(l) == 1 && l[0].State.IsRunning() { - container = l[0] - break - } - time.Sleep(10 * time.Millisecond) - } - }) + container := waitContainerStart(t, 10*time.Second) setTimeout(t, "Reading container's id timed out", 10*time.Second, func() { buf := make([]byte, 1024) @@ -508,16 +538,8 @@ func TestAttachDetach(t *testing.T) { <-ch }) - pty, err := container.GetPtyMaster() - if err != nil { - t.Fatal(err) - } - - state, err := term.MakeRaw(pty.Fd()) - if err != nil { - t.Fatal(err) - } - defer term.RestoreTerminal(pty.Fd(), state) + state := setRaw(t, container) + defer unsetRaw(t, container, state) stdin, stdinPipe = io.Pipe() stdout, stdoutPipe = io.Pipe() From 2e6a958612d65a0665a9396372fe82706987d085 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 10:03:36 -0800 Subject: [PATCH 073/162] Fix TestAttachDetachTruncatedID (behavior + tty issue) --- integration/commands_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/integration/commands_test.go b/integration/commands_test.go index fa8f25836c..90333a052c 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -601,7 +601,10 @@ func TestAttachDetachTruncatedID(t *testing.T) { } }) - container := globalRuntime.List()[0] + container := waitContainerStart(t, 10*time.Second) + + state := setRaw(t, container) + defer unsetRaw(t, container, state) stdin, stdinPipe = io.Pipe() stdout, stdoutPipe = io.Pipe() @@ -626,17 +629,16 @@ func TestAttachDetachTruncatedID(t *testing.T) { }) setTimeout(t, "Escape sequence timeout", 5*time.Second, func() { - stdinPipe.Write([]byte{16, 17}) - if err := stdinPipe.Close(); err != nil { - t.Fatal(err) - } + stdinPipe.Write([]byte{16}) + time.Sleep(100 * time.Millisecond) + stdinPipe.Write([]byte{17}) }) - closeWrap(stdin, stdinPipe, stdout, stdoutPipe) // wait for CmdRun to return setTimeout(t, "Waiting for CmdAttach timed out", 15*time.Second, func() { <-ch }) + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) time.Sleep(500 * time.Millisecond) if !container.State.IsRunning() { From 2ec1146679598837cd8bab62dc672bcda2a9610c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 10:17:04 -0800 Subject: [PATCH 074/162] Remove an unit test from integrations test --- integration/commands_test.go | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/integration/commands_test.go b/integration/commands_test.go index 90333a052c..ba9399218a 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -774,25 +774,6 @@ func TestCmdLogs(t *testing.T) { } } -// Expected behaviour: using / as a bind mount source should throw an error -func TestRunErrorBindMountRootSource(t *testing.T) { - - cli := docker.NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr) - defer cleanup(globalEngine, t) - - c := make(chan struct{}) - go func() { - defer close(c) - if err := cli.CmdRun("-v", "/:/tmp", unitTestImageID, "echo 'should fail'"); err == nil { - t.Fatal("should have failed to run when using / as a source for the bind mount") - } - }() - - setTimeout(t, "CmdRun timed out", 5*time.Second, func() { - <-c - }) -} - // Expected behaviour: error out when attempting to bind mount non-existing source paths func TestRunErrorBindNonExistingSource(t *testing.T) { @@ -802,6 +783,7 @@ func TestRunErrorBindNonExistingSource(t *testing.T) { c := make(chan struct{}) go func() { defer close(c) + // This check is made at runtime, can't be "unit tested" if err := cli.CmdRun("-v", "/i/dont/exist:/tmp", unitTestImageID, "echo 'should fail'"); err == nil { t.Fatal("should have failed to run when using /i/dont/exist as a source for the bind mount") } From 86c00be180f1e6831ca426576a55f5106f156448 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 10:17:25 -0800 Subject: [PATCH 075/162] Fix behavior of tty tests --- integration/commands_test.go | 47 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/integration/commands_test.go b/integration/commands_test.go index ba9399218a..50cd230ce9 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -337,7 +337,8 @@ func TestRunDisconnect(t *testing.T) { }) } -// Expected behaviour: the process dies when the client disconnects +// Expected behaviour: the process stay alive when the client disconnects +// but the client detaches. func TestRunDisconnectTty(t *testing.T) { stdin, stdinPipe := io.Pipe() @@ -348,29 +349,20 @@ func TestRunDisconnectTty(t *testing.T) { c1 := make(chan struct{}) go func() { + defer close(c1) // We're simulating a disconnect so the return value doesn't matter. What matters is the // fact that CmdRun returns. if err := cli.CmdRun("-i", "-t", unitTestImageID, "/bin/cat"); err != nil { utils.Debugf("Error CmdRun: %s", err) } - - close(c1) }() - setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { - for { - // Client disconnect after run -i should keep stdin out in TTY mode - l := globalRuntime.List() - if len(l) == 1 && l[0].State.IsRunning() { - break - } - time.Sleep(10 * time.Millisecond) - } - }) + container := waitContainerStart(t, 10*time.Second) + + state := setRaw(t, container) + defer unsetRaw(t, container, state) // Client disconnect after run -i should keep stdin out in TTY mode - container := globalRuntime.List()[0] - setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) @@ -382,8 +374,12 @@ func TestRunDisconnectTty(t *testing.T) { t.Fatal(err) } + // wait for CmdRun to return + setTimeout(t, "Waiting for CmdRun timed out", 5*time.Second, func() { + <-c1 + }) + // In tty mode, we expect the process to stay alive even after client's stdin closes. - // Do not wait for run to finish // Give some time to monitor to do his thing container.WaitTimeout(500 * time.Millisecond) @@ -473,27 +469,28 @@ func TestRunDetach(t *testing.T) { cli.CmdRun("-i", "-t", unitTestImageID, "cat") }() + container := waitContainerStart(t, 10*time.Second) + + state := setRaw(t, container) + defer unsetRaw(t, container, state) + setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 150); err != nil { t.Fatal(err) } }) - container := globalRuntime.List()[0] - setTimeout(t, "Escape sequence timeout", 5*time.Second, func() { - stdinPipe.Write([]byte{16, 17}) - if err := stdinPipe.Close(); err != nil { - t.Fatal(err) - } + stdinPipe.Write([]byte{16}) + time.Sleep(100 * time.Millisecond) + stdinPipe.Write([]byte{17}) }) - closeWrap(stdin, stdinPipe, stdout, stdoutPipe) - // wait for CmdRun to return setTimeout(t, "Waiting for CmdRun timed out", 15*time.Second, func() { <-ch }) + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) time.Sleep(500 * time.Millisecond) if !container.State.IsRunning() { @@ -594,6 +591,7 @@ func TestAttachDetachTruncatedID(t *testing.T) { cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) + // Discard the CmdRun output go stdout.Read(make([]byte, 1024)) setTimeout(t, "Starting container timed out", 2*time.Second, func() { if err := cli.CmdRun("-i", "-t", "-d", unitTestImageID, "cat"); err != nil { @@ -759,6 +757,7 @@ func TestRunAutoRemove(t *testing.T) { } func TestCmdLogs(t *testing.T) { + t.Skip("Test not impemented") cli := docker.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) From ca98434a45a1c836aa69d67d8cc12b8e95e5722f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 29 Nov 2013 11:06:35 -0800 Subject: [PATCH 076/162] Search for repo first before image id --- tags.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tags.go b/tags.go index 21c13bdfb2..92c32b1ff5 100644 --- a/tags.go +++ b/tags.go @@ -66,20 +66,18 @@ func (store *TagStore) Reload() error { } func (store *TagStore) LookupImage(name string) (*Image, error) { - img, err := store.graph.Get(name) + // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else + // (so we can pass all errors here) + repos, tag := utils.ParseRepositoryTag(name) + if tag == "" { + tag = DEFAULTTAG + } + img, err := store.GetImage(repos, tag) if err != nil { - // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else - // (so we can pass all errors here) - repos, tag := utils.ParseRepositoryTag(name) - if tag == "" { - tag = DEFAULTTAG - } - if i, err := store.GetImage(repos, tag); err != nil { + return nil, err + } else if img == nil { + if img, err = store.graph.Get(name); err != nil { return nil, err - } else if i == nil { - return nil, fmt.Errorf("Image does not exist: %s", name) - } else { - img = i } } return img, nil From 34353e782e1cdbd6aae078b3e660875e703d35ff Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 10:32:52 -0800 Subject: [PATCH 077/162] Reduce the timeout for restart/stop --- integration/server_test.go | 4 ++-- term/term.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/server_test.go b/integration/server_test.go index 24e109ab76..494e23fef3 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -183,11 +183,11 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerRestart(id, 150); err != nil { + if err := srv.ContainerRestart(id, 15); err != nil { t.Fatal(err) } - if err := srv.ContainerStop(id, 150); err != nil { + if err := srv.ContainerStop(id, 15); err != nil { t.Fatal(err) } diff --git a/term/term.go b/term/term.go index 50425a8602..f7d9930ad0 100644 --- a/term/term.go +++ b/term/term.go @@ -9,7 +9,7 @@ import ( ) var ( - ErrInvalidState = errors.New("Invlide terminal state") + ErrInvalidState = errors.New("Invlid terminal state") ) type State struct { From ab35aef6b5b15c7c2d7aff4315025563b93ee379 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 13:43:37 -0800 Subject: [PATCH 078/162] Add unit test to check bind / server side --- integration/api_test.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/integration/api_test.go b/integration/api_test.go index bd6280af8b..3d5a2e42e0 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -743,6 +743,43 @@ func TestPostContainersStart(t *testing.T) { containerKill(eng, containerID, t) } +// Expected behaviour: using / as a bind mount source should throw an error +func TestRunErrorBindMountRootSource(t *testing.T) { + eng := NewTestEngine(t) + defer mkRuntimeFromEngine(eng, t).Nuke() + srv := mkServerFromEngine(eng, t) + + containerID := createTestContainer( + eng, + &docker.Config{ + Image: unitTestImageID, + Cmd: []string{"/bin/cat"}, + OpenStdin: true, + }, + t, + ) + + hostConfigJSON, err := json.Marshal(&docker.HostConfig{ + Binds: []string{"/:/tmp"}, + }) + + req, err := http.NewRequest("POST", "/containers/"+containerID+"/start", bytes.NewReader(hostConfigJSON)) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Content-Type", "application/json") + + r := httptest.NewRecorder() + if err := docker.ServeRequest(srv, docker.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusInternalServerError { + containerKill(eng, containerID, t) + t.Fatal("should have failed to run when using / as a source for the bind mount") + } +} + func TestPostContainersStop(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() From db7c55ba7f97358f3a6de2b83c8e3d1827667446 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 29 Nov 2013 14:13:00 -0800 Subject: [PATCH 079/162] Catch SIGQUIT for cleanup --- server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index 725b964a70..ae9578fbf9 100644 --- a/server.go +++ b/server.go @@ -38,7 +38,7 @@ func init() { // jobInitApi runs the remote api server `srv` as a daemon, // Only one api server can run at the same time - this is enforced by a pidfile. -// The signals SIGINT and SIGTERM are intercepted for cleanup. +// The signals SIGINT, SIGQUIT and SIGTERM are intercepted for cleanup. func jobInitApi(job *engine.Job) string { job.Logf("Creating server") srv, err := NewServer(job.Eng, ConfigFromJob(job)) @@ -53,7 +53,7 @@ func jobInitApi(job *engine.Job) string { } job.Logf("Setting up signal traps") c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Signal(syscall.SIGTERM)) + signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) go func() { sig := <-c log.Printf("Received signal '%v', exiting\n", sig) From a37b155384401e8b2bed635251440b0feb6bcdbc Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 29 Nov 2013 15:14:36 -0800 Subject: [PATCH 080/162] Split auth on first colon --- auth/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth/auth.go b/auth/auth.go index 9acd00df61..62d8e7f522 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -63,7 +63,7 @@ func decodeAuth(authStr string) (string, string, error) { if n > decLen { return "", "", fmt.Errorf("Something went wrong decoding auth config") } - arr := strings.Split(string(decoded), ":") + arr := strings.SplitN(string(decoded), ":", 2) if len(arr) != 2 { return "", "", fmt.Errorf("Invalid auth configuration file") } From fe72f15e4ab25cc6e96c76d2da2c379569756843 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Fri, 29 Nov 2013 22:20:59 -0200 Subject: [PATCH 081/162] go fmt. result of `gofmt -w -s .` without vendors. --- integration/graph_test.go | 26 +++++++++++++------------- registry/registry.go | 2 +- utils_test.go | 30 +++++++++++++++--------------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/integration/graph_test.go b/integration/graph_test.go index 118ec17863..eec4c5c7dc 100644 --- a/integration/graph_test.go +++ b/integration/graph_test.go @@ -287,19 +287,19 @@ func assertNImages(graph *docker.Graph, t *testing.T, n int) { } func tempGraph(t *testing.T) (*docker.Graph, graphdriver.Driver) { - tmp, err := ioutil.TempDir("", "docker-graph-") - if err != nil { - t.Fatal(err) - } - driver, err := graphdriver.New(tmp) - if err != nil { - t.Fatal(err) - } - graph, err := docker.NewGraph(tmp, driver) - if err != nil { - t.Fatal(err) - } - return graph, driver + tmp, err := ioutil.TempDir("", "docker-graph-") + if err != nil { + t.Fatal(err) + } + driver, err := graphdriver.New(tmp) + if err != nil { + t.Fatal(err) + } + graph, err := docker.NewGraph(tmp, driver) + if err != nil { + t.Fatal(err) + } + return graph, driver } func nukeGraph(graph *docker.Graph) { diff --git a/registry/registry.go b/registry/registry.go index d3d9f2be54..6aea458e99 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -47,7 +47,7 @@ func pingRegistryEndpoint(endpoint string) error { if err != nil { return err } - defer resp.Body.Close() + defer resp.Body.Close() if resp.Header.Get("X-Docker-Registry-Version") == "" { return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") diff --git a/utils_test.go b/utils_test.go index f4fa80a5f1..4b8cfba39f 100644 --- a/utils_test.go +++ b/utils_test.go @@ -1,24 +1,24 @@ package docker import ( - "io" "archive/tar" "bytes" + "io" ) func fakeTar() (io.Reader, error) { - content := []byte("Hello world!\n") - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { - hdr := new(tar.Header) - hdr.Size = int64(len(content)) - hdr.Name = name - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - tw.Write([]byte(content)) - } - tw.Close() - return buf, nil + content := []byte("Hello world!\n") + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { + hdr := new(tar.Header) + hdr.Size = int64(len(content)) + hdr.Name = name + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + tw.Write([]byte(content)) + } + tw.Close() + return buf, nil } From a4f8a2494b0fb755db52c68cf61ddc8ff52d2965 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Nov 2013 07:37:03 +0000 Subject: [PATCH 082/162] Engine: integer job status, improved stream API * Jobs return an integer status instead of a string * Status convention mimics unix process execution: 0=success, 1=generic error, 127="no such command" * Stdout and Stderr support multiple thread-safe data receivers and ring buffer filtering --- api.go | 12 ++- engine/engine.go | 10 +- engine/engine_test.go | 7 +- engine/helpers_test.go | 18 +--- engine/job.go | 147 ++++++++++-------------------- engine/streams.go | 166 ++++++++++++++++++++++++++++++++++ integration/api_test.go | 4 + integration/container_test.go | 4 +- integration/runtime_test.go | 2 +- integration/server_test.go | 2 +- integration/utils_test.go | 2 +- server.go | 72 +++++++++------ 12 files changed, 291 insertions(+), 155 deletions(-) create mode 100644 engine/streams.go diff --git a/api.go b/api.go index aadd79e3c8..1708420e12 100644 --- a/api.go +++ b/api.go @@ -1,6 +1,8 @@ package docker import ( + "bufio" + "bytes" "code.google.com/p/go.net/websocket" "encoding/base64" "encoding/json" @@ -565,12 +567,18 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r job.SetenvList("Dns", defaultDns) } // Read container ID from the first line of stdout - job.StdoutParseString(&out.ID) + job.Stdout.AddString(&out.ID) // Read warnings from stderr - job.StderrParseLines(&out.Warnings, 0) + warnings := &bytes.Buffer{} + job.Stderr.Add(warnings) if err := job.Run(); err != nil { return err } + // Parse warnings from stderr + scanner := bufio.NewScanner(warnings) + for scanner.Scan() { + out.Warnings = append(out.Warnings, scanner.Text()) + } if job.GetenvInt("Memory") > 0 && !srv.runtime.capabilities.MemoryLimit { log.Println("WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.") out.Warnings = append(out.Warnings, "Your kernel does not support memory limit capabilities. Limitation discarded.") diff --git a/engine/engine.go b/engine/engine.go index edd04fc5c0..5a411e8cc2 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -9,7 +9,7 @@ import ( "strings" ) -type Handler func(*Job) string +type Handler func(*Job) Status var globalHandlers map[string]Handler @@ -99,10 +99,12 @@ func (eng *Engine) Job(name string, args ...string) *Job { Eng: eng, Name: name, Args: args, - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, + Stdin: NewInput(), + Stdout: NewOutput(), + Stderr: NewOutput(), } + job.Stdout.Add(utils.NopWriteCloser(os.Stdout)) + job.Stderr.Add(utils.NopWriteCloser(os.Stderr)) handler, exists := eng.handlers[name] if exists { job.handler = handler diff --git a/engine/engine_test.go b/engine/engine_test.go index fdc0b0ec7f..f877a3e4d5 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -38,8 +38,9 @@ func TestJob(t *testing.T) { t.Fatalf("job1.handler should be empty") } - h := func(j *Job) string { - return j.Name + h := func(j *Job) Status { + j.Printf("%s\n", j.Name) + return 42 } eng.Register("dummy2", h) @@ -49,7 +50,7 @@ func TestJob(t *testing.T) { t.Fatalf("job2.handler shouldn't be nil") } - if job2.handler(job2) != job2.Name { + if job2.handler(job2) != 42 { t.Fatalf("handler dummy2 was not found in job2") } } diff --git a/engine/helpers_test.go b/engine/helpers_test.go index 5b1c0baf60..488529fc7f 100644 --- a/engine/helpers_test.go +++ b/engine/helpers_test.go @@ -1,32 +1,18 @@ package engine import ( - "fmt" "github.com/dotcloud/docker/utils" - "io/ioutil" - "runtime" - "strings" "testing" ) var globalTestID string func newTestEngine(t *testing.T) *Engine { - // Use the caller function name as a prefix. - // This helps trace temp directories back to their test. - pc, _, _, _ := runtime.Caller(1) - callerLongName := runtime.FuncForPC(pc).Name() - parts := strings.Split(callerLongName, ".") - callerShortName := parts[len(parts)-1] - if globalTestID == "" { - globalTestID = utils.RandomString()[:4] - } - prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, callerShortName) - root, err := ioutil.TempDir("", prefix) + tmp, err := utils.TestDirectory("") if err != nil { t.Fatal(err) } - eng, err := New(root) + eng, err := New(tmp) if err != nil { t.Fatal(err) } diff --git a/engine/job.go b/engine/job.go index 365c94e06b..066a144664 100644 --- a/engine/job.go +++ b/engine/job.go @@ -1,16 +1,13 @@ package engine import ( - "bufio" "bytes" "encoding/json" "fmt" "io" - "io/ioutil" - "os" "strconv" "strings" - "sync" + "time" ) // A job is the fundamental unit of work in the docker engine. @@ -31,126 +28,75 @@ type Job struct { Name string Args []string env []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - handler func(*Job) string - status string + Stdout *Output + Stderr *Output + Stdin *Input + handler Handler + status Status + end time.Time onExit []func() } +type Status int + +const ( + StatusOK Status = 0 + StatusErr Status = 1 + StatusNotFound Status = 127 +) + // Run executes the job and blocks until the job completes. // If the job returns a failure status, an error is returned // which includes the status. func (job *Job) Run() error { - defer func() { - var wg sync.WaitGroup - for _, f := range job.onExit { - wg.Add(1) - go func(f func()) { - f() - wg.Done() - }(f) - } - wg.Wait() - }() - if job.Stdout != nil && job.Stdout != os.Stdout { - job.Stdout = io.MultiWriter(job.Stdout, os.Stdout) - } - if job.Stderr != nil && job.Stderr != os.Stderr { - job.Stderr = io.MultiWriter(job.Stderr, os.Stderr) + // FIXME: make this thread-safe + // FIXME: implement wait + if !job.end.IsZero() { + return fmt.Errorf("%s: job has already completed", job.Name) } + // Log beginning and end of the job job.Eng.Logf("+job %s", job.CallString()) defer func() { job.Eng.Logf("-job %s%s", job.CallString(), job.StatusString()) }() + var errorMessage string + job.Stderr.AddString(&errorMessage) if job.handler == nil { - job.status = "command not found" + job.Errorf("%s: command not found", job.Name) + job.status = 127 } else { job.status = job.handler(job) + job.end = time.Now() } - if job.status != "0" { - return fmt.Errorf("%s: %s", job.Name, job.status) + // Wait for all background tasks to complete + if err := job.Stdout.Close(); err != nil { + return err + } + if err := job.Stderr.Close(); err != nil { + return err + } + if job.status != 0 { + return fmt.Errorf("%s: %s", job.Name, errorMessage) } return nil } -func (job *Job) StdoutParseLines(dst *[]string, limit int) { - job.parseLines(job.StdoutPipe(), dst, limit) -} - -func (job *Job) StderrParseLines(dst *[]string, limit int) { - job.parseLines(job.StderrPipe(), dst, limit) -} - -func (job *Job) parseLines(src io.Reader, dst *[]string, limit int) { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - scanner := bufio.NewScanner(src) - for scanner.Scan() { - // If the limit is reached, flush the rest of the source and return - if limit > 0 && len(*dst) >= limit { - io.Copy(ioutil.Discard, src) - return - } - line := scanner.Text() - // Append the line (with delimitor removed) - *dst = append(*dst, line) - } - }() - job.onExit = append(job.onExit, wg.Wait) -} - -func (job *Job) StdoutParseString(dst *string) { - lines := make([]string, 0, 1) - job.StdoutParseLines(&lines, 1) - job.onExit = append(job.onExit, func() { - if len(lines) >= 1 { - *dst = lines[0] - } - }) -} - -func (job *Job) StderrParseString(dst *string) { - lines := make([]string, 0, 1) - job.StderrParseLines(&lines, 1) - job.onExit = append(job.onExit, func() { *dst = lines[0] }) -} - -func (job *Job) StdoutPipe() io.ReadCloser { - r, w := io.Pipe() - job.Stdout = w - job.onExit = append(job.onExit, func() { w.Close() }) - return r -} - -func (job *Job) StderrPipe() io.ReadCloser { - r, w := io.Pipe() - job.Stderr = w - job.onExit = append(job.onExit, func() { w.Close() }) - return r -} - func (job *Job) CallString() string { return fmt.Sprintf("%s(%s)", job.Name, strings.Join(job.Args, ", ")) } func (job *Job) StatusString() string { - // FIXME: if a job returns the empty string, it will be printed - // as not having returned. - // (this only affects String which is a convenience function). - if job.status != "" { - var okerr string - if job.status == "0" { - okerr = "OK" - } else { - okerr = "ERR" - } - return fmt.Sprintf(" = %s (%s)", okerr, job.status) + // If the job hasn't completed, status string is empty + if job.end.IsZero() { + return "" } - return "" + var okerr string + if job.status == StatusOK { + okerr = "OK" + } else { + okerr = "ERR" + } + return fmt.Sprintf(" = %s (%d)", okerr, job.status) } // String returns a human-readable description of `job` @@ -338,5 +284,8 @@ func (job *Job) Printf(format string, args ...interface{}) (n int, err error) { func (job *Job) Errorf(format string, args ...interface{}) (n int, err error) { return fmt.Fprintf(job.Stderr, format, args...) - +} + +func (job *Job) Error(err error) (int, error) { + return fmt.Fprintf(job.Stderr, "%s", err) } diff --git a/engine/streams.go b/engine/streams.go new file mode 100644 index 0000000000..697407f1f4 --- /dev/null +++ b/engine/streams.go @@ -0,0 +1,166 @@ +package engine + +import ( + "bufio" + "container/ring" + "fmt" + "io" + "sync" +) + +type Output struct { + sync.Mutex + dests []io.Writer + tasks sync.WaitGroup +} + +// NewOutput returns a new Output object with no destinations attached. +// Writing to an empty Output will cause the written data to be discarded. +func NewOutput() *Output { + return &Output{} +} + +// Add attaches a new destination to the Output. Any data subsequently written +// to the output will be written to the new destination in addition to all the others. +// This method is thread-safe. +// FIXME: Add cannot fail +func (o *Output) Add(dst io.Writer) error { + o.Mutex.Lock() + defer o.Mutex.Unlock() + o.dests = append(o.dests, dst) + return nil +} + +// AddPipe creates an in-memory pipe with io.Pipe(), adds its writing end as a destination, +// and returns its reading end for consumption by the caller. +// This is a rough equivalent similar to Cmd.StdoutPipe() in the standard os/exec package. +// This method is thread-safe. +func (o *Output) AddPipe() (io.Reader, error) { + r, w := io.Pipe() + o.Add(w) + return r, nil +} + +// AddTail starts a new goroutine which will read all subsequent data written to the output, +// line by line, and append the last `n` lines to `dst`. +func (o *Output) AddTail(dst *[]string, n int) error { + src, err := o.AddPipe() + if err != nil { + return err + } + o.tasks.Add(1) + go func() { + defer o.tasks.Done() + Tail(src, n, dst) + }() + return nil +} + +// AddString starts a new goroutine which will read all subsequent data written to the output, +// line by line, and store the last line into `dst`. +func (o *Output) AddString(dst *string) error { + src, err := o.AddPipe() + if err != nil { + return err + } + o.tasks.Add(1) + go func() { + defer o.tasks.Done() + lines := make([]string, 0, 1) + Tail(src, 1, &lines) + if len(lines) == 0 { + *dst = "" + } else { + *dst = lines[0] + } + }() + return nil +} + +// Write writes the same data to all registered destinations. +// This method is thread-safe. +func (o *Output) Write(p []byte) (n int, err error) { + o.Mutex.Lock() + defer o.Mutex.Unlock() + var firstErr error + for _, dst := range o.dests { + _, err := dst.Write(p) + if err != nil && firstErr == nil { + firstErr = err + } + } + return len(p), err +} + +// Close unregisters all destinations and waits for all background +// AddTail and AddString tasks to complete. +// The Close method of each destination is called if it exists. +func (o *Output) Close() error { + o.Mutex.Lock() + defer o.Mutex.Unlock() + var firstErr error + for _, dst := range o.dests { + if closer, ok := dst.(io.WriteCloser); ok { + err := closer.Close() + if err != nil && firstErr == nil { + firstErr = err + } + } + } + o.tasks.Wait() + return firstErr +} + +type Input struct { + src io.Reader + sync.Mutex +} + +// NewInput returns a new Input object with no source attached. +// Reading to an empty Input will return io.EOF. +func NewInput() *Input { + return &Input{} +} + +// Read reads from the input in a thread-safe way. +func (i *Input) Read(p []byte) (n int, err error) { + i.Mutex.Lock() + defer i.Mutex.Unlock() + if i.src == nil { + return 0, io.EOF + } + return i.src.Read(p) +} + +// Add attaches a new source to the input. +// Add can only be called once per input. Subsequent calls will +// return an error. +func (i *Input) Add(src io.Reader) error { + i.Mutex.Lock() + defer i.Mutex.Unlock() + if i.src != nil { + return fmt.Errorf("Maximum number of sources reached: 1") + } + i.src = src + return nil +} + +// Tail reads from `src` line per line, and returns the last `n` lines as an array. +// A ring buffer is used to only store `n` lines at any time. +func Tail(src io.Reader, n int, dst *[]string) { + scanner := bufio.NewScanner(src) + r := ring.New(n) + for scanner.Scan() { + if n == 0 { + continue + } + r.Value = scanner.Text() + r = r.Next() + } + r.Do(func(v interface{}) { + if v == nil { + return + } + *dst = append(*dst, v.(string)) + }) +} diff --git a/integration/api_test.go b/integration/api_test.go index 3d5a2e42e0..896831c3d0 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -304,6 +304,10 @@ func TestGetContainersJSON(t *testing.T) { Cmd: []string{"echo", "test"}, }, t) + if containerID == "" { + t.Fatalf("Received empty container ID") + } + req, err := http.NewRequest("GET", "/containers/json?all=1", nil) if err != nil { t.Fatal(err) diff --git a/integration/container_test.go b/integration/container_test.go index 05eb48728c..6cd72c8608 100644 --- a/integration/container_test.go +++ b/integration/container_test.go @@ -499,7 +499,7 @@ func TestCreateVolume(t *testing.T) { t.Fatal(err) } var id string - jobCreate.StdoutParseString(&id) + jobCreate.Stdout.AddString(&id) if err := jobCreate.Run(); err != nil { t.Fatal(err) } @@ -1502,7 +1502,7 @@ func TestOnlyLoopbackExistsWhenUsingDisableNetworkOption(t *testing.T) { t.Fatal(err) } var id string - jobCreate.StdoutParseString(&id) + jobCreate.Stdout.AddString(&id) if err := jobCreate.Run(); err != nil { t.Fatal(err) } diff --git a/integration/runtime_test.go b/integration/runtime_test.go index 1ab6d0a080..7074a14ce9 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -390,7 +390,7 @@ func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *doc jobCreate.SetenvList("Cmd", []string{"sh", "-c", cmd}) jobCreate.SetenvList("PortSpecs", []string{fmt.Sprintf("%s/%s", strPort, proto)}) jobCreate.SetenvJson("ExposedPorts", ep) - jobCreate.StdoutParseString(&id) + jobCreate.Stdout.AddString(&id) if err := jobCreate.Run(); err != nil { t.Fatal(err) } diff --git a/integration/server_test.go b/integration/server_test.go index 494e23fef3..3e85effe8f 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -224,7 +224,7 @@ func TestRunWithTooLowMemoryLimit(t *testing.T) { job.Setenv("CpuShares", "1000") job.SetenvList("Cmd", []string{"/bin/cat"}) var id string - job.StdoutParseString(&id) + job.Stdout.AddString(&id) if err := job.Run(); err == nil { t.Errorf("Memory limit is smaller than the allowed limit. Container creation should've failed!") } diff --git a/integration/utils_test.go b/integration/utils_test.go index 1f47c45382..2feaf25396 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -46,7 +46,7 @@ func createNamedTestContainer(eng *engine.Engine, config *docker.Config, f utils if err := job.ImportEnv(config); err != nil { f.Fatal(err) } - job.StdoutParseString(&shortId) + job.Stdout.AddString(&shortId) if err := job.Run(); err != nil { f.Fatal(err) } diff --git a/server.go b/server.go index 725b964a70..ae6ccd4f40 100644 --- a/server.go +++ b/server.go @@ -39,15 +39,18 @@ func init() { // jobInitApi runs the remote api server `srv` as a daemon, // Only one api server can run at the same time - this is enforced by a pidfile. // The signals SIGINT and SIGTERM are intercepted for cleanup. -func jobInitApi(job *engine.Job) string { +func jobInitApi(job *engine.Job) engine.Status { job.Logf("Creating server") + // FIXME: ImportEnv deprecates ConfigFromJob srv, err := NewServer(job.Eng, ConfigFromJob(job)) if err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } if srv.runtime.config.Pidfile != "" { job.Logf("Creating pidfile") if err := utils.CreatePidFile(srv.runtime.config.Pidfile); err != nil { + // FIXME: do we need fatal here instead of returning a job error? log.Fatal(err) } } @@ -68,18 +71,21 @@ func jobInitApi(job *engine.Job) string { job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP) } if err := job.Eng.Register("create", srv.ContainerCreate); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } if err := job.Eng.Register("start", srv.ContainerStart); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } if err := job.Eng.Register("serveapi", srv.ListenAndServe); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } - return "0" + return engine.StatusOK } -func (srv *Server) ListenAndServe(job *engine.Job) string { +func (srv *Server) ListenAndServe(job *engine.Job) engine.Status { protoAddrs := job.Args chErrors := make(chan error, len(protoAddrs)) for _, protoAddr := range protoAddrs { @@ -94,7 +100,8 @@ func (srv *Server) ListenAndServe(job *engine.Job) string { log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") } default: - return "Invalid protocol format." + job.Errorf("Invalid protocol format.") + return engine.StatusErr } go func() { // FIXME: merge Server.ListenAndServe with ListenAndServe @@ -104,10 +111,11 @@ func (srv *Server) ListenAndServe(job *engine.Job) string { for i := 0; i < len(protoAddrs); i += 1 { err := <-chErrors if err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } } - return "0" + return engine.StatusOK } func (srv *Server) DockerVersion() APIVersion { @@ -1260,19 +1268,22 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return nil } -func (srv *Server) ContainerCreate(job *engine.Job) string { +func (srv *Server) ContainerCreate(job *engine.Job) engine.Status { var name string if len(job.Args) == 1 { name = job.Args[0] } else if len(job.Args) > 1 { - return fmt.Sprintf("Usage: %s ", job.Name) + job.Printf("Usage: %s", job.Name) + return engine.StatusErr } var config Config if err := job.ExportEnv(&config); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } if config.Memory != 0 && config.Memory < 524288 { - return "Minimum memory limit allowed is 512k" + job.Errorf("Minimum memory limit allowed is 512k") + return engine.StatusErr } if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit { config.Memory = 0 @@ -1287,9 +1298,11 @@ func (srv *Server) ContainerCreate(job *engine.Job) string { if tag == "" { tag = DEFAULTTAG } - return fmt.Sprintf("No such image: %s (tag: %s)", config.Image, tag) + job.Errorf("No such image: %s (tag: %s)", config.Image, tag) + return engine.StatusErr } - return err.Error() + job.Error(err) + return engine.StatusErr } srv.LogEvent("create", container.ID, srv.runtime.repositories.ImageName(container.Image)) // FIXME: this is necessary because runtime.Create might return a nil container @@ -1301,7 +1314,7 @@ func (srv *Server) ContainerCreate(job *engine.Job) string { for _, warning := range buildWarnings { job.Errorf("%s\n", warning) } - return "0" + return engine.StatusOK } func (srv *Server) ContainerRestart(name string, t int) error { @@ -1619,22 +1632,25 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { return nil } -func (srv *Server) ContainerStart(job *engine.Job) string { +func (srv *Server) ContainerStart(job *engine.Job) engine.Status { if len(job.Args) < 1 { - return fmt.Sprintf("Usage: %s container_id", job.Name) + job.Errorf("Usage: %s container_id", job.Name) + return engine.StatusErr } name := job.Args[0] runtime := srv.runtime container := runtime.Get(name) if container == nil { - return fmt.Sprintf("No such container: %s", name) + job.Errorf("No such container: %s", name) + return engine.StatusErr } // If no environment was set, then no hostconfig was passed. if len(job.Environ()) > 0 { var hostConfig HostConfig if err := job.ExportEnv(&hostConfig); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } // Validate the HostConfig binds. Make sure that: // 1) the source of a bind mount isn't / @@ -1647,29 +1663,33 @@ func (srv *Server) ContainerStart(job *engine.Job) string { // refuse to bind mount "/" to the container if source == "/" { - return fmt.Sprintf("Invalid bind mount '%s' : source can't be '/'", bind) + job.Errorf("Invalid bind mount '%s' : source can't be '/'", bind) + return engine.StatusErr } // ensure the source exists on the host _, err := os.Stat(source) if err != nil && os.IsNotExist(err) { - return fmt.Sprintf("Invalid bind mount '%s' : source doesn't exist", bind) + job.Errorf("Invalid bind mount '%s' : source doesn't exist", bind) + return engine.StatusErr } } // Register any links from the host config before starting the container // FIXME: we could just pass the container here, no need to lookup by name again. if err := srv.RegisterLinks(name, &hostConfig); err != nil { - return err.Error() + job.Error(err) + return engine.StatusErr } container.hostConfig = &hostConfig container.ToDisk() } if err := container.Start(); err != nil { - return fmt.Sprintf("Cannot start container %s: %s", name, err) + job.Errorf("Cannot start container %s: %s", name, err) + return engine.StatusErr } srv.LogEvent("start", container.ID, runtime.repositories.ImageName(container.Image)) - return "0" + return engine.StatusOK } func (srv *Server) ContainerStop(name string, t int) error { From 3553a803e3a474207296f5542182ecbc569ca1d8 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Nov 2013 07:38:59 +0000 Subject: [PATCH 083/162] Engine: better testing of streams and of basic engine primitives. Coverage=81.2% --- engine/engine_test.go | 47 ++++++++ engine/job_test.go | 80 +++++++++++++ engine/streams_test.go | 252 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 379 insertions(+) create mode 100644 engine/job_test.go create mode 100644 engine/streams_test.go diff --git a/engine/engine_test.go b/engine/engine_test.go index f877a3e4d5..793867f50a 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -1,6 +1,9 @@ package engine import ( + "io/ioutil" + "os" + "path" "testing" ) @@ -54,3 +57,47 @@ func TestJob(t *testing.T) { t.Fatalf("handler dummy2 was not found in job2") } } + +func TestEngineRoot(t *testing.T) { + tmp, err := ioutil.TempDir("", "docker-test-TestEngineCreateDir") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + dir := path.Join(tmp, "dir") + eng, err := New(dir) + if err != nil { + t.Fatal(err) + } + if st, err := os.Stat(dir); err != nil { + t.Fatal(err) + } else if !st.IsDir() { + t.Fatalf("engine.New() created something other than a directory at %s", dir) + } + if r := eng.Root(); r != dir { + t.Fatalf("Expected: %v\nReceived: %v", dir, r) + } +} + +func TestEngineString(t *testing.T) { + eng1 := newTestEngine(t) + defer os.RemoveAll(eng1.Root()) + eng2 := newTestEngine(t) + defer os.RemoveAll(eng2.Root()) + s1 := eng1.String() + s2 := eng2.String() + if eng1 == eng2 { + t.Fatalf("Different engines should have different names (%v == %v)", s1, s2) + } +} + +func TestEngineLogf(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + input := "Test log line" + if n, err := eng.Logf("%s\n", input); err != nil { + t.Fatal(err) + } else if n < len(input) { + t.Fatalf("Test: Logf() should print at least as much as the input\ninput=%d\nprinted=%d", len(input), n) + } +} diff --git a/engine/job_test.go b/engine/job_test.go new file mode 100644 index 0000000000..50d882c44b --- /dev/null +++ b/engine/job_test.go @@ -0,0 +1,80 @@ +package engine + +import ( + "os" + "testing" +) + +func TestJobStatusOK(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + eng.Register("return_ok", func(job *Job) Status { return StatusOK }) + err := eng.Job("return_ok").Run() + if err != nil { + t.Fatalf("Expected: err=%v\nReceived: err=%v", nil, err) + } +} + +func TestJobStatusErr(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + eng.Register("return_err", func(job *Job) Status { return StatusErr }) + err := eng.Job("return_err").Run() + if err == nil { + t.Fatalf("When a job returns StatusErr, Run() should return an error") + } +} + +func TestJobStatusNotFound(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + eng.Register("return_not_found", func(job *Job) Status { return StatusNotFound }) + err := eng.Job("return_not_found").Run() + if err == nil { + t.Fatalf("When a job returns StatusNotFound, Run() should return an error") + } +} + +func TestJobStdoutString(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + // FIXME: test multiple combinations of output and status + eng.Register("say_something_in_stdout", func(job *Job) Status { + job.Printf("Hello world\n") + return StatusOK + }) + + job := eng.Job("say_something_in_stdout") + var output string + if err := job.Stdout.AddString(&output); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { + t.Fatal(err) + } + if expectedOutput := "Hello world"; output != expectedOutput { + t.Fatalf("Stdout last line:\nExpected: %v\nReceived: %v", expectedOutput, output) + } +} + +func TestJobStderrString(t *testing.T) { + eng := newTestEngine(t) + defer os.RemoveAll(eng.Root()) + // FIXME: test multiple combinations of output and status + eng.Register("say_something_in_stderr", func(job *Job) Status { + job.Errorf("Warning, something might happen\nHere it comes!\nOh no...\nSomething happened\n") + return StatusOK + }) + + job := eng.Job("say_something_in_stderr") + var output string + if err := job.Stderr.AddString(&output); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { + t.Fatal(err) + } + if expectedOutput := "Something happened"; output != expectedOutput { + t.Fatalf("Stderr last line:\nExpected: %v\nReceived: %v", expectedOutput, output) + } +} diff --git a/engine/streams_test.go b/engine/streams_test.go new file mode 100644 index 0000000000..d7faf229af --- /dev/null +++ b/engine/streams_test.go @@ -0,0 +1,252 @@ +package engine + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "strings" + "testing" +) + +func TestOutputAddString(t *testing.T) { + var testInputs = [][2]string{ + { + "hello, world!", + "hello, world!", + }, + + { + "One\nTwo\nThree", + "Three", + }, + + { + "", + "", + }, + + { + "A line\nThen another nl-terminated line\n", + "Then another nl-terminated line", + }, + + { + "A line followed by an empty line\n\n", + "", + }, + } + for _, testData := range testInputs { + input := testData[0] + expectedOutput := testData[1] + o := NewOutput() + var output string + if err := o.AddString(&output); err != nil { + t.Error(err) + } + if n, err := o.Write([]byte(input)); err != nil { + t.Error(err) + } else if n != len(input) { + t.Errorf("Expected %d, got %d", len(input), n) + } + o.Close() + if output != expectedOutput { + t.Errorf("Last line is not stored as return string.\nInput: '%s'\nExpected: '%s'\nGot: '%s'", input, expectedOutput, output) + } + } +} + +type sentinelWriteCloser struct { + calledWrite bool + calledClose bool +} + +func (w *sentinelWriteCloser) Write(p []byte) (int, error) { + w.calledWrite = true + return len(p), nil +} + +func (w *sentinelWriteCloser) Close() error { + w.calledClose = true + return nil +} + +func TestOutputAddClose(t *testing.T) { + o := NewOutput() + var s sentinelWriteCloser + if err := o.Add(&s); err != nil { + t.Fatal(err) + } + if err := o.Close(); err != nil { + t.Fatal(err) + } + // Write data after the output is closed. + // Write should succeed, but no destination should receive it. + if _, err := o.Write([]byte("foo bar")); err != nil { + t.Fatal(err) + } + if !s.calledClose { + t.Fatal("Output.Close() didn't close the destination") + } +} + +func TestOutputAddPipe(t *testing.T) { + var testInputs = []string{ + "hello, world!", + "One\nTwo\nThree", + "", + "A line\nThen another nl-terminated line\n", + "A line followed by an empty line\n\n", + } + for _, input := range testInputs { + expectedOutput := input + o := NewOutput() + r, err := o.AddPipe() + if err != nil { + t.Fatal(err) + } + go func(o *Output) { + if n, err := o.Write([]byte(input)); err != nil { + t.Error(err) + } else if n != len(input) { + t.Errorf("Expected %d, got %d", len(input), n) + } + if err := o.Close(); err != nil { + t.Error(err) + } + }(o) + output, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(output) != expectedOutput { + t.Errorf("Last line is not stored as return string.\nExpected: '%s'\nGot: '%s'", expectedOutput, output) + } + } +} + +func TestTail(t *testing.T) { + var tests = make(map[string][][]string) + tests["hello, world!"] = [][]string{ + {}, + {"hello, world!"}, + {"hello, world!"}, + {"hello, world!"}, + } + tests["One\nTwo\nThree"] = [][]string{ + {}, + {"Three"}, + {"Two", "Three"}, + {"One", "Two", "Three"}, + } + for input, outputs := range tests { + for n, expectedOutput := range outputs { + var output []string + Tail(strings.NewReader(input), n, &output) + if fmt.Sprintf("%v", output) != fmt.Sprintf("%v", expectedOutput) { + t.Errorf("Tail n=%d returned wrong result.\nExpected: '%s'\nGot : '%s'", expectedOutput, output) + } + } + } +} + +func TestOutputAddTail(t *testing.T) { + var tests = make(map[string][][]string) + tests["hello, world!"] = [][]string{ + {}, + {"hello, world!"}, + {"hello, world!"}, + {"hello, world!"}, + } + tests["One\nTwo\nThree"] = [][]string{ + {}, + {"Three"}, + {"Two", "Three"}, + {"One", "Two", "Three"}, + } + for input, outputs := range tests { + for n, expectedOutput := range outputs { + o := NewOutput() + var output []string + if err := o.AddTail(&output, n); err != nil { + t.Error(err) + } + if n, err := o.Write([]byte(input)); err != nil { + t.Error(err) + } else if n != len(input) { + t.Errorf("Expected %d, got %d", len(input), n) + } + o.Close() + if fmt.Sprintf("%v", output) != fmt.Sprintf("%v", expectedOutput) { + t.Errorf("Tail(%d) returned wrong result.\nExpected: %v\nGot: %v", n, expectedOutput, output) + } + } + } +} + +func lastLine(txt string) string { + scanner := bufio.NewScanner(strings.NewReader(txt)) + var lastLine string + for scanner.Scan() { + lastLine = scanner.Text() + } + return lastLine +} + +func TestOutputAdd(t *testing.T) { + o := NewOutput() + b := &bytes.Buffer{} + o.Add(b) + input := "hello, world!" + if n, err := o.Write([]byte(input)); err != nil { + t.Fatal(err) + } else if n != len(input) { + t.Fatalf("Expected %d, got %d", len(input), n) + } + if output := b.String(); output != input { + t.Fatal("Received wrong data from Add.\nExpected: '%s'\nGot: '%s'", input, output) + } +} + +func TestInputAddEmpty(t *testing.T) { + i := NewInput() + var b bytes.Buffer + if err := i.Add(&b); err != nil { + t.Fatal(err) + } + data, err := ioutil.ReadAll(i) + if err != nil { + t.Fatal(err) + } + if len(data) > 0 { + t.Fatalf("Read from empty input shoul yield no data") + } +} + +func TestInputAddTwo(t *testing.T) { + i := NewInput() + var b1 bytes.Buffer + // First add should succeed + if err := i.Add(&b1); err != nil { + t.Fatal(err) + } + var b2 bytes.Buffer + // Second add should fail + if err := i.Add(&b2); err == nil { + t.Fatalf("Adding a second source should return an error") + } +} + +func TestInputAddNotEmpty(t *testing.T) { + i := NewInput() + b := bytes.NewBufferString("hello world\nabc") + expectedResult := b.String() + i.Add(b) + result, err := ioutil.ReadAll(i) + if err != nil { + t.Fatal(err) + } + if string(result) != expectedResult { + t.Fatalf("Expected: %v\nReceived: %v", expectedResult, result) + } +} From 35d54c665575d48954db9422702c0324f00ebc62 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Nov 2013 02:28:56 +0000 Subject: [PATCH 084/162] Fix a bug in Output.Write, and improve testing coverage of error cases. --- engine/streams.go | 2 +- engine/streams_test.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/engine/streams.go b/engine/streams.go index 697407f1f4..ff7049074a 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -89,7 +89,7 @@ func (o *Output) Write(p []byte) (n int, err error) { firstErr = err } } - return len(p), err + return len(p), firstErr } // Close unregisters all destinations and waits for all background diff --git a/engine/streams_test.go b/engine/streams_test.go index d7faf229af..108c9850a5 100644 --- a/engine/streams_test.go +++ b/engine/streams_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "io" "io/ioutil" "strings" "testing" @@ -208,6 +209,27 @@ func TestOutputAdd(t *testing.T) { } } +func TestOutputWriteError(t *testing.T) { + o := NewOutput() + buf := &bytes.Buffer{} + o.Add(buf) + r, w := io.Pipe() + input := "Hello there" + expectedErr := fmt.Errorf("This is an error") + r.CloseWithError(expectedErr) + o.Add(w) + n, err := o.Write([]byte(input)) + if err != expectedErr { + t.Fatalf("Output.Write() should return the first error encountered, if any") + } + if buf.String() != input { + t.Fatalf("Output.Write() should attempt write on all destinations, even after encountering an error") + } + if n != len(input) { + t.Fatalf("Output.Write() should return the size of the input if it successfully writes to at least one destination") + } +} + func TestInputAddEmpty(t *testing.T) { i := NewInput() var b bytes.Buffer From 65db62619c8ae61c9735cb2a433adac7731256a5 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 29 Nov 2013 16:45:20 -0800 Subject: [PATCH 085/162] Only return published ports for docker port --- commands.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/commands.go b/commands.go index 846a66c7a4..ce12f8a607 100644 --- a/commands.go +++ b/commands.go @@ -763,16 +763,12 @@ func (cli *DockerCli) CmdPort(args ...string) error { return err } - if frontends, exists := out.NetworkSettings.Ports[Port(port+"/"+proto)]; exists { - if frontends == nil { - fmt.Fprintf(cli.out, "%s\n", port) - } else { - for _, frontend := range frontends { - fmt.Fprintf(cli.out, "%s:%s\n", frontend.HostIp, frontend.HostPort) - } + if frontends, exists := out.NetworkSettings.Ports[Port(port+"/"+proto)]; exists && frontends != nil { + for _, frontend := range frontends { + fmt.Fprintf(cli.out, "%s:%s\n", frontend.HostIp, frontend.HostPort) } } else { - return fmt.Errorf("Error: No private port '%s' allocated on %s", cmd.Arg(1), cmd.Arg(0)) + return fmt.Errorf("Error: No public port '%s' published for %s", cmd.Arg(1), cmd.Arg(0)) } return nil } From a6c9a332d040a98c5cf4325491fc8d9aa7e5475b Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Fri, 29 Nov 2013 22:53:20 -0200 Subject: [PATCH 086/162] fixed some `go vet` issues. --- archive/changes_test.go | 70 +++++++++++++++--------------- engine/env_test.go | 6 +-- graphdriver/devmapper/deviceset.go | 6 +-- integration/runtime_test.go | 4 +- network.go | 2 +- server.go | 2 +- 6 files changed, 45 insertions(+), 45 deletions(-) diff --git a/archive/changes_test.go b/archive/changes_test.go index a840c691ff..714ab71e2d 100644 --- a/archive/changes_test.go +++ b/archive/changes_test.go @@ -247,7 +247,7 @@ func TestChangesDirsMutated(t *testing.T) { } if changes[i].Path == expectedChanges[i].Path { if changes[i] != expectedChanges[i] { - t.Fatalf("Wrong change for %s, expected %s, got %d\n", changes[i].Path, changes[i].String(), expectedChanges[i].String()) + t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String()) } } else if changes[i].Path < expectedChanges[i].Path { t.Fatalf("unexpected change %s\n", changes[i].String()) @@ -261,45 +261,45 @@ func TestApplyLayer(t *testing.T) { t.Skip("Skipping TestApplyLayer due to known failures") // Disable this for now as it is broken return - src, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } - createSampleDir(t, src) - dst := src + "-copy" - if err := copyDir(src, dst); err != nil { - t.Fatal(err) - } - mutateSampleDir(t, dst) + // src, err := ioutil.TempDir("", "docker-changes-test") + // if err != nil { + // t.Fatal(err) + // } + // createSampleDir(t, src) + // dst := src + "-copy" + // if err := copyDir(src, dst); err != nil { + // t.Fatal(err) + // } + // mutateSampleDir(t, dst) - changes, err := ChangesDirs(dst, src) - if err != nil { - t.Fatal(err) - } + // changes, err := ChangesDirs(dst, src) + // if err != nil { + // t.Fatal(err) + // } - layer, err := ExportChanges(dst, changes) - if err != nil { - t.Fatal(err) - } + // layer, err := ExportChanges(dst, changes) + // if err != nil { + // t.Fatal(err) + // } - layerCopy, err := NewTempArchive(layer, "") - if err != nil { - t.Fatal(err) - } + // layerCopy, err := NewTempArchive(layer, "") + // if err != nil { + // t.Fatal(err) + // } - if err := ApplyLayer(src, layerCopy); err != nil { - t.Fatal(err) - } + // if err := ApplyLayer(src, layerCopy); err != nil { + // t.Fatal(err) + // } - changes2, err := ChangesDirs(src, dst) - if err != nil { - t.Fatal(err) - } + // changes2, err := ChangesDirs(src, dst) + // if err != nil { + // t.Fatal(err) + // } - if len(changes2) != 0 { - t.Fatalf("Unexpected differences after re applying mutation: %v", changes) - } + // if len(changes2) != 0 { + // t.Fatalf("Unexpected differences after re applying mutation: %v", changes) + // } - os.RemoveAll(src) - os.RemoveAll(dst) + // os.RemoveAll(src) + // os.RemoveAll(dst) } diff --git a/engine/env_test.go b/engine/env_test.go index 343da31cec..24c5992dd0 100644 --- a/engine/env_test.go +++ b/engine/env_test.go @@ -37,16 +37,16 @@ func TestSetenvBool(t *testing.T) { job := mkJob(t, "dummy") job.SetenvBool("foo", true) if val := job.GetenvBool("foo"); !val { - t.Fatalf("GetenvBool returns incorrect value: %b", val) + t.Fatalf("GetenvBool returns incorrect value: %t", val) } job.SetenvBool("bar", false) if val := job.GetenvBool("bar"); val { - t.Fatalf("GetenvBool returns incorrect value: %b", val) + t.Fatalf("GetenvBool returns incorrect value: %t", val) } if val := job.GetenvBool("nonexistent"); val { - t.Fatalf("GetenvBool returns incorrect value: %b", val) + t.Fatalf("GetenvBool returns incorrect value: %t", val) } } diff --git a/graphdriver/devmapper/deviceset.go b/graphdriver/devmapper/deviceset.go index 77d865ad99..a65c02f87e 100644 --- a/graphdriver/devmapper/deviceset.go +++ b/graphdriver/devmapper/deviceset.go @@ -172,7 +172,7 @@ func (devices *DeviceSet) saveMetadata() error { return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) } if err := osRename(tmpFile.Name(), devices.jsonFile()); err != nil { - return fmt.Errorf("Error committing metadata file", err) + return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) } if devices.NewTransactionId != devices.TransactionId { @@ -439,11 +439,11 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { hasMetadata := devices.hasImage("metadata") if !doInit && !hasData { - return fmt.Errorf("Looback data file not found %s") + return fmt.Error("Looback data file not found") } if !doInit && !hasMetadata { - return fmt.Errorf("Looback metadata file not found %s") + return fmt.Error("Looback metadata file not found") } createdLoopback := !hasData || !hasMetadata diff --git a/integration/runtime_test.go b/integration/runtime_test.go index 1ab6d0a080..fda8a03829 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -127,7 +127,7 @@ func setupBaseImage() { job.SetenvBool("Autorestart", false) job.Setenv("BridgeIface", unitTestNetworkBridge) if err := job.Run(); err != nil { - log.Fatalf("Unable to create a runtime for tests:", err) + log.Fatalf("Unable to create a runtime for tests: %s", err) } srv := mkServerFromEngine(eng, log.New(os.Stderr, "", 0)) @@ -173,7 +173,7 @@ func spawnGlobalDaemon() { func GetTestImage(runtime *docker.Runtime) *docker.Image { imgs, err := runtime.Graph().Map() if err != nil { - log.Fatalf("Unable to get the test image:", err) + log.Fatalf("Unable to get the test image: %s", err) } for _, image := range imgs { if image.ID == unitTestImageID { diff --git a/network.go b/network.go index 1397de0557..dd23dd48c1 100644 --- a/network.go +++ b/network.go @@ -580,7 +580,7 @@ type Nat struct { } func (n *Nat) String() string { - return fmt.Sprintf("%s:%d:%d/%s", n.Binding.HostIp, n.Binding.HostPort, n.Port.Port(), n.Port.Proto()) + return fmt.Sprintf("%s:%s:%s/%s", n.Binding.HostIp, n.Binding.HostPort, n.Port.Port(), n.Port.Proto()) } // Release: Network cleanup - release all resources diff --git a/server.go b/server.go index 725b964a70..c1e6c7d87b 100644 --- a/server.go +++ b/server.go @@ -386,8 +386,8 @@ func (srv *Server) recursiveLoad(address, tmpImageDir string) error { imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json")) if err != nil { - return err utils.Debugf("Error reading json", err) + return err } layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar")) From e8437e8fcffae3661b9decdc1c576083c0de5219 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Sat, 30 Nov 2013 01:02:09 -0200 Subject: [PATCH 087/162] using `errors.New` instead `fmt.Error`. --- graphdriver/devmapper/deviceset.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/graphdriver/devmapper/deviceset.go b/graphdriver/devmapper/deviceset.go index a65c02f87e..f2e99d9ed7 100644 --- a/graphdriver/devmapper/deviceset.go +++ b/graphdriver/devmapper/deviceset.go @@ -2,6 +2,7 @@ package devmapper import ( "encoding/json" + "errors" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -439,11 +440,11 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { hasMetadata := devices.hasImage("metadata") if !doInit && !hasData { - return fmt.Error("Looback data file not found") + return errors.New("Looback data file not found") } if !doInit && !hasMetadata { - return fmt.Error("Looback metadata file not found") + return errors.New("Looback metadata file not found") } createdLoopback := !hasData || !hasMetadata From 56ab9cb0d5169401616acc7a396e9f4af00ed670 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Sat, 30 Nov 2013 00:07:53 -0800 Subject: [PATCH 088/162] Minor fixes based on discussions on #2693 * Volume exports ./bundles instead of root directory * Documents build using docker-docs instead of docker:docs * Bundles directory is created before running build or docs --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1447a4ed12..dd992946a4 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,18 @@ default: build -build: +build: bundles docker build -t docker . - docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh binary + docker run -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh binary doc: - cd docs && docker build -t docker:docs . && docker run -p 8000:8000 docker:docs + cd docs && docker build -t docker-docs . && docker run -p 8000:8000 docker-docs -test: - docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test +test: bundles + docker run -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh test shell: docker run -privileged -i -t docker bash + +bundles: + mkdir bundles + From 39aac21db47b3bcbf651008267ad1142f1213103 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sat, 30 Nov 2013 10:58:52 -0500 Subject: [PATCH 089/162] Updated and simplified the PG example --- docs/sources/examples/postgresql_service.rst | 39 +++++++------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/docs/sources/examples/postgresql_service.rst b/docs/sources/examples/postgresql_service.rst index 82ca8b59ca..c57b3c41f2 100644 --- a/docs/sources/examples/postgresql_service.rst +++ b/docs/sources/examples/postgresql_service.rst @@ -7,26 +7,18 @@ PostgreSQL Service ================== +.. include:: example_header.inc + .. note:: A shorter version of `this blog post`_. -.. note:: - - As of version 0.5.2, Docker requires root privileges to run. - You have to either manually adjust your system configuration (permissions on - /var/run/docker.sock or sudo config), or prefix `docker` with `sudo`. Check - `this thread`_ for details. - .. _this blog post: http://zaiste.net/2013/08/docker_postgresql_how_to/ -.. _this thread: https://groups.google.com/forum/?fromgroups#!topic/docker-club/P3xDLqmLp0E Installing PostgreSQL on Docker ------------------------------- -For clarity I won't be showing command output. - -Run an interactive shell in Docker container. +Run an interactive shell in a Docker container. .. code-block:: bash @@ -38,19 +30,17 @@ Update its dependencies. apt-get update -Install ``python-software-properties``. +Install ``python-software-properties``, ``software-properties-common``, ``wget`` and ``vim``. .. code-block:: bash - apt-get -y install python-software-properties - apt-get -y install software-properties-common + apt-get -y install python-software-properties software-properties-common wget vim Add PostgreSQL's repository. It contains the most recent stable release -of PostgreSQL i.e. ``9.3``. +of PostgreSQL, ``9.3``. .. code-block:: bash - apt-get -y install wget wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list apt-get update @@ -78,15 +68,14 @@ role. Adjust PostgreSQL configuration so that remote connections to the database are possible. Make sure that inside -``/etc/postgresql/9.3/main/pg_hba.conf`` you have following line (you will need -to install an editor, e.g. ``apt-get install vim``): +``/etc/postgresql/9.3/main/pg_hba.conf`` you have following line: .. code-block:: bash host all all 0.0.0.0/0 md5 Additionaly, inside ``/etc/postgresql/9.3/main/postgresql.conf`` -uncomment ``listen_addresses`` so it is as follows: +uncomment ``listen_addresses`` like so: .. code-block:: bash @@ -104,14 +93,14 @@ Exit. exit -Create an image and assign it a name. ```` is in the -Bash prompt; you can also locate it using ``docker ps -a``. +Create an image from our container and assign it a name. The ```` +is in the Bash prompt; you can also locate it using ``docker ps -a``. .. code-block:: bash sudo docker commit /postgresql -Finally, run PostgreSQL server via ``docker``. +Finally, run the PostgreSQL server via ``docker``. .. code-block:: bash @@ -121,8 +110,8 @@ Finally, run PostgreSQL server via ``docker``. -D /var/lib/postgresql/9.3/main \ -c config_file=/etc/postgresql/9.3/main/postgresql.conf') -Connect the PostgreSQL server using ``psql`` (You will need postgres installed -on the machine. For ubuntu, use something like +Connect the PostgreSQL server using ``psql`` (You will need PostgreSQL installed +on the machine. For Ubuntu, use something like ``sudo apt-get install postgresql``). .. code-block:: bash @@ -140,7 +129,7 @@ As before, create roles or databases if needed. docker=# CREATE DATABASE foo OWNER=docker; CREATE DATABASE -Additionally, publish your newly created image on Docker Index. +Additionally, publish your newly created image on the Docker Index. .. code-block:: bash From fea432bdf542a469ae9b5b114b6858a7bc56b323 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Sat, 30 Nov 2013 16:28:52 -0200 Subject: [PATCH 090/162] fixed `Looback` typo. --- graphdriver/devmapper/deviceset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graphdriver/devmapper/deviceset.go b/graphdriver/devmapper/deviceset.go index f2e99d9ed7..a2a88040f5 100644 --- a/graphdriver/devmapper/deviceset.go +++ b/graphdriver/devmapper/deviceset.go @@ -440,11 +440,11 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { hasMetadata := devices.hasImage("metadata") if !doInit && !hasData { - return errors.New("Looback data file not found") + return errors.New("Loopback data file not found") } if !doInit && !hasMetadata { - return errors.New("Looback metadata file not found") + return errors.New("Loopback metadata file not found") } createdLoopback := !hasData || !hasMetadata From 5c5f6709017575b0b090cb5f76b75e7e733ffa2b Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 1 Dec 2013 15:11:10 -0700 Subject: [PATCH 091/162] Update to Go 1.2 officially, now that it is released --- Dockerfile | 2 +- hack/PACKAGERS.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index adba9f215a..961681d57e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ run apt-get install -y -q mercurial run apt-get install -y -q build-essential libsqlite3-dev # Install Go -run curl -s https://go.googlecode.com/files/go1.2rc5.src.tar.gz | tar -v -C /usr/local -xz +run curl -s https://go.googlecode.com/files/go1.2.src.tar.gz | tar -v -C /usr/local -xz env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor run cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std diff --git a/hack/PACKAGERS.md b/hack/PACKAGERS.md index 1ac061c01e..90f99a799f 100644 --- a/hack/PACKAGERS.md +++ b/hack/PACKAGERS.md @@ -36,7 +36,7 @@ To build docker, you will need the following system dependencies * An amd64 machine * A recent version of git and mercurial -* Go version 1.2rc1 or later (see notes below regarding using Go 1.1.2 and dynbinary) +* Go version 1.2 or later (see notes below regarding using Go 1.1.2 and dynbinary) * SQLite version 3.7.9 or later * A clean checkout of the source must be added to a valid Go [workspace](http://golang.org/doc/code.html#Workspaces) under the path *src/github.com/dotcloud/docker*. @@ -91,8 +91,8 @@ You would do the users of your distro a disservice and "void the docker warranty A good comparison is Busybox: all distros package it as a statically linked binary, because it just makes sense. Docker is the same way. -If you *must* have a non-static Docker binary, or require Go 1.1.2 (since Go 1.2 is not yet officially -released at the time of this writing), please use: +If you *must* have a non-static Docker binary, or require Go 1.1.2 (since Go 1.2 is still freshly released +at the time of this writing), please use: ```bash ./hack/make.sh dynbinary From c30e2dc28c53664a4ad241a682355332af3371d2 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 1 Dec 2013 21:20:35 -0700 Subject: [PATCH 092/162] Remove "-v" from "go test" (since it's easy to add back via TESTFLAGS) --- hack/make/dyntest | 2 +- hack/make/test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/make/dyntest b/hack/make/dyntest index 756b038207..a4ab2e456b 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -34,7 +34,7 @@ bundle_test() { # Run the tests with the optional $TESTFLAGS. export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION - go test -v -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS + go test -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS ); then TESTS_FAILED+=("$test_dir") sleep 1 # give it a second, so observers watching can take note diff --git a/hack/make/test b/hack/make/test index 361f731d70..eab8c18c84 100644 --- a/hack/make/test +++ b/hack/make/test @@ -27,7 +27,7 @@ bundle_test() { go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS # Run the tests with the optional $TESTFLAGS. - go test -v -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS + go test -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS ); then TESTS_FAILED+=("$test_dir") sleep 1 # give it a second, so observers watching can take note From 7f1a91121c3f77ca833f43693018f6cce32b96bf Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 1 Dec 2013 21:31:28 -0700 Subject: [PATCH 093/162] Update Dockerfile with all-caps INSTRUCTIONS (as explained in docs as being "convention": http://docs.docker.io/en/latest/use/builder/#format) --- Dockerfile | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 961681d57e..cc5a19276f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,52 +24,52 @@ # docker-version 0.6.1 -from ubuntu:12.04 -maintainer Solomon Hykes +FROM ubuntu:12.04 +MAINTAINER Solomon Hykes # Build dependencies -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -run apt-get update -run apt-get install -y -q curl -run apt-get install -y -q git -run apt-get install -y -q mercurial -run apt-get install -y -q build-essential libsqlite3-dev +RUN echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list +RUN apt-get update +RUN apt-get install -y -q curl +RUN apt-get install -y -q git +RUN apt-get install -y -q mercurial +RUN apt-get install -y -q build-essential libsqlite3-dev # Install Go -run curl -s https://go.googlecode.com/files/go1.2.src.tar.gz | tar -v -C /usr/local -xz -env PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin -env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor -run cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std +RUN curl -s https://go.googlecode.com/files/go1.2.src.tar.gz | tar -v -C /usr/local -xz +ENV PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin +ENV GOPATH /go:/go/src/github.com/dotcloud/docker/vendor +RUN cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std # Ubuntu stuff -run apt-get install -y -q ruby1.9.3 rubygems libffi-dev -run gem install --no-rdoc --no-ri fpm -run apt-get install -y -q reprepro dpkg-sig +RUN apt-get install -y -q ruby1.9.3 rubygems libffi-dev +RUN gem install --no-rdoc --no-ri fpm +RUN apt-get install -y -q reprepro dpkg-sig -run apt-get install -y -q python-pip -run pip install s3cmd==1.1.0-beta3 -run pip install python-magic==0.4.6 -run /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg +RUN apt-get install -y -q python-pip +RUN pip install s3cmd==1.1.0-beta3 +RUN pip install python-magic==0.4.6 +RUN /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg # Runtime dependencies -run apt-get install -y -q iptables -run apt-get install -y -q lxc -run apt-get install -y -q aufs-tools +RUN apt-get install -y -q iptables +RUN apt-get install -y -q lxc +RUN apt-get install -y -q aufs-tools # Get lvm2 source for compiling statically -run git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout v2_02_103 +RUN git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout v2_02_103 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags # note: we can't use "git clone -b" above because it requires at least git 1.7.10 to be able to use that on a tag instead of a branch and we only have 1.7.9.5 # Compile and install lvm2 -run cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper +RUN cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL -volume /var/lib/docker -workdir /go/src/github.com/dotcloud/docker +VOLUME /var/lib/docker +WORKDIR /go/src/github.com/dotcloud/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers -entrypoint ["hack/dind"] +ENTRYPOINT ["hack/dind"] # Upload docker source -add . /go/src/github.com/dotcloud/docker +ADD . /go/src/github.com/dotcloud/docker From 26cf8b9aff1cffa7c3e3ae7e720c960ff545ee78 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Mon, 2 Dec 2013 15:52:45 +1000 Subject: [PATCH 094/162] check on the client side that there is a Dockerfile, so we don't upload a huge stack of files, only to realise we can't do anything --- commands.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/commands.go b/commands.go index 5d2ea1806c..fb4b11cfc1 100644 --- a/commands.go +++ b/commands.go @@ -195,6 +195,10 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if _, err := os.Stat(cmd.Arg(0)); err != nil { return err } + filename := path.Join(cmd.Arg(0), "Dockerfile") + if _, err = os.Stat(filename); os.IsNotExist(err) { + return fmt.Errorf("no Dockerfile found in %s", cmd.Arg(0)) + } context, err = archive.Tar(cmd.Arg(0), archive.Uncompressed) } var body io.Reader From dbb47f63ab637a4766f72eac55f66493cf1c6f5b Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 1 Dec 2013 22:13:05 -0700 Subject: [PATCH 095/162] Add udev rules files for hiding the docker loopback devices from udisks This prevents them from showing up on the desktop in a window manager, for example. --- contrib/udev/80-docker.rules | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contrib/udev/80-docker.rules diff --git a/contrib/udev/80-docker.rules b/contrib/udev/80-docker.rules new file mode 100644 index 0000000000..f934c01757 --- /dev/null +++ b/contrib/udev/80-docker.rules @@ -0,0 +1,3 @@ +# hide docker's loopback devices from udisks, and thus from user desktops +SUBSYSTEM=="block", ENV{DM_NAME}=="docker-*", ENV{UDISKS_PRESENTATION_HIDE}="1", ENV{UDISKS_IGNORE}="1" +SUBSYSTEM=="block", DEVPATH=="/devices/virtual/block/loop*", ATTR{loop/backing_file}=="/var/lib/docker/*", ENV{UDISKS_PRESENTATION_HIDE}="1", ENV{UDISKS_IGNORE}="1" From b9ad0c9f742a3e48ad06770abb362048306c763e Mon Sep 17 00:00:00 2001 From: Ben Sargent Date: Mon, 2 Dec 2013 16:52:12 +0000 Subject: [PATCH 096/162] Added format of Volumes and Binds parameters to create and start * Added sample Volumes parameter to container create API. * Added PortBindings and PublishAllPorts parameter to container start API * Added note to container start about Binds needed to be declared as Volumes during container create. --- docs/sources/api/docker_remote_api_v1.7.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/sources/api/docker_remote_api_v1.7.rst b/docs/sources/api/docker_remote_api_v1.7.rst index 251b162250..1dccf889d7 100644 --- a/docs/sources/api/docker_remote_api_v1.7.rst +++ b/docs/sources/api/docker_remote_api_v1.7.rst @@ -132,7 +132,9 @@ Create a container ], "Dns":null, "Image":"base", - "Volumes":{}, + "Volumes":{ + "/tmp": {} + }, "VolumesFrom":"", "WorkingDir":"" @@ -361,8 +363,12 @@ Start a container { "Binds":["/tmp:/tmp"], - "LxcConf":{"lxc.utsname":"docker"} + "LxcConf":{"lxc.utsname":"docker"}, + "PortBindings":null + "PublishAllPorts":false } + + Binds need to reference Volumes that were defined during container creation. **Example response**: From b04c6466cdc89a107879af7a22b0917006b38730 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Sat, 12 Oct 2013 14:14:52 +0200 Subject: [PATCH 097/162] Make docker build return exit code of build step If a command during build fails, `docker build` now returns with the exit code of that command. This makes it necessary to change the build api endpoint to return a json object stream. --- api.go | 10 +- buildfile.go | 45 ++++++-- commands.go | 46 ++------ docker/docker.go | 5 +- docs/sources/api/docker_remote_api.rst | 5 + docs/sources/api/docker_remote_api_v1.7.rst | 6 +- integration/buildfile_test.go | 121 +++++++++++++++----- integration/commands_test.go | 7 +- utils/utils.go | 5 +- 9 files changed, 168 insertions(+), 82 deletions(-) diff --git a/api.go b/api.go index 1708420e12..92ab727869 100644 --- a/api.go +++ b/api.go @@ -960,9 +960,17 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ return err } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache, rm) + if version > 1.6 { + w.Header().Set("Content-Type", "application/json") + } + sf := utils.NewStreamFormatter(version > 1.6) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache, rm, sf) id, err := b.Build(context) if err != nil { + if sf.Used() { + w.Write(sf.FormatError(err)) + return nil + } return fmt.Errorf("Error build: %s", err) } if repoName != "" { diff --git a/buildfile.go b/buildfile.go index 3075812436..15c27ecd7c 100644 --- a/buildfile.go +++ b/buildfile.go @@ -37,6 +37,7 @@ type buildFile struct { tmpImages map[string]struct{} out io.Writer + sf *utils.StreamFormatter } func (b *buildFile) clearTmp(containers map[string]struct{}) { @@ -52,7 +53,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil, nil, true); err != nil { + if err := b.srv.ImagePull(remote, tag, b.out, b.sf, nil, nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) @@ -100,7 +101,11 @@ func (b *buildFile) CmdRun(args string) error { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return err } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") + if b.sf.Used() { + b.out.Write(b.sf.FormatStatus("", " ---> Using cache")) + } else { + fmt.Fprintf(b.out, " ---> Using cache\n") + } utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return nil @@ -376,8 +381,11 @@ func (b *buildFile) run() (string, error) { return "", err } b.tmpContainers[c.ID] = struct{}{} - fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID)) - + if b.sf.Used() { + b.out.Write(b.sf.FormatStatus("", " ---> Running in %s", utils.TruncateID(c.ID))) + } else { + fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID)) + } // override the entry point that may have been picked up from the base image c.Path = b.config.Cmd[0] c.Args = b.config.Cmd[1:] @@ -403,7 +411,11 @@ func (b *buildFile) run() (string, error) { // Wait for it to finish if ret := c.Wait(); ret != 0 { - return "", fmt.Errorf("The command %v returned a non-zero code: %d", b.config.Cmd, ret) + err := &utils.JSONError{ + Message: fmt.Sprintf("The command %v returned a non-zero code: %d", b.config.Cmd, ret), + Code: ret, + } + return "", err } return c.ID, nil @@ -424,7 +436,11 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return err } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") + if b.sf.Used() { + b.out.Write(b.sf.FormatStatus("", " ---> Using cache")) + } else { + fmt.Fprintf(b.out, " ---> Using cache\n") + } utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return nil @@ -441,7 +457,11 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { fmt.Fprintf(b.out, " ---> [Warning] %s\n", warning) } b.tmpContainers[container.ID] = struct{}{} - fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) + if b.sf.Used() { + b.out.Write(b.sf.FormatStatus("", " ---> Running in %s", utils.TruncateID(container.ID))) + } else { + fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) + } id = container.ID if err := container.EnsureMounted(); err != nil { return err @@ -507,22 +527,22 @@ func (b *buildFile) Build(context io.Reader) (string, error) { method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) if !exists { - fmt.Fprintf(b.out, "# Skipping unknown instruction %s\n", strings.ToUpper(instruction)) + b.out.Write(b.sf.FormatStatus("", "# Skipping unknown instruction %s", strings.ToUpper(instruction))) continue } stepN += 1 - fmt.Fprintf(b.out, "Step %d : %s %s\n", stepN, strings.ToUpper(instruction), arguments) + b.out.Write(b.sf.FormatStatus("", "Step %d : %s %s", stepN, strings.ToUpper(instruction), arguments)) ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() if ret != nil { return "", ret.(error) } - fmt.Fprintf(b.out, " ---> %v\n", utils.TruncateID(b.image)) + b.out.Write(b.sf.FormatStatus("", " ---> %s", utils.TruncateID(b.image))) } if b.image != "" { - fmt.Fprintf(b.out, "Successfully built %s\n", utils.TruncateID(b.image)) + b.out.Write(b.sf.FormatStatus("", "Successfully built %s", utils.TruncateID(b.image))) if b.rm { b.clearTmp(b.tmpContainers) } @@ -531,7 +551,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occurred during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache, rm bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache, rm bool, sf *utils.StreamFormatter) BuildFile { return &buildFile{ runtime: srv.runtime, srv: srv, @@ -542,5 +562,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache, rm bool) Bu verbose: verbose, utilizeCache: utilizeCache, rm: rm, + sf: sf, } } diff --git a/commands.go b/commands.go index 5d2ea1806c..ef95e32c33 100644 --- a/commands.go +++ b/commands.go @@ -220,42 +220,16 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if *rm { v.Set("rm", "1") } - req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) - if err != nil { - return err - } + + headers := http.Header(make(map[string][]string)) if context != nil { - req.Header.Set("Content-Type", "application/tar") + headers.Set("Content-Type", "application/tar") } - dial, err := net.Dial(cli.proto, cli.addr) - if err != nil { - return err + err = cli.stream("POST", fmt.Sprintf("/build?%s", v.Encode()), body, cli.out, headers) + if jerr, ok := err.(*utils.JSONError); ok { + return &utils.StatusError{Status: jerr.Message, StatusCode: jerr.Code} } - clientconn := httputil.NewClientConn(dial, nil) - resp, err := clientconn.Do(req) - defer clientconn.Close() - if err != nil { - return err - } - defer resp.Body.Close() - // Check for errors - if resp.StatusCode < 200 || resp.StatusCode >= 400 { - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - if len(body) == 0 { - return fmt.Errorf("Error: %s", http.StatusText(resp.StatusCode)) - } - return fmt.Errorf("Error: %s", body) - } - - // Output the result - if _, err := io.Copy(cli.out, resp.Body); err != nil { - return err - } - - return nil + return err } // 'docker login': login / register a user to registry service. @@ -699,7 +673,7 @@ func (cli *DockerCli) CmdInspect(args ...string) error { } fmt.Fprintf(cli.out, "]") if status != 0 { - return &utils.StatusError{Status: status} + return &utils.StatusError{StatusCode: status} } return nil } @@ -1584,7 +1558,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { return err } if status != 0 { - return &utils.StatusError{Status: status} + return &utils.StatusError{StatusCode: status} } return nil @@ -2167,7 +2141,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { } } if status != 0 { - return &utils.StatusError{Status: status} + return &utils.StatusError{StatusCode: status} } return nil } diff --git a/docker/docker.go b/docker/docker.go index 83df5c2171..2404ab3173 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -100,7 +100,10 @@ func main() { protoAddrParts := strings.SplitN(flHosts[0], "://", 2) if err := docker.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { if sterr, ok := err.(*utils.StatusError); ok { - os.Exit(sterr.Status) + if sterr.Status != "" { + log.Println(sterr.Status) + } + os.Exit(sterr.StatusCode) } log.Fatal(err) } diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index d5cb1a44a9..5ef031ec03 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -138,6 +138,11 @@ What's new This URI no longer exists. The ``images -viz`` output is now generated in the client, using the ``/images/json`` data. +.. http:post:: /build + + **New!** This endpoint now returns build status as json stream. In case + of a build error, it returns the exit status of the failed command. + v1.6 **** diff --git a/docs/sources/api/docker_remote_api_v1.7.rst b/docs/sources/api/docker_remote_api_v1.7.rst index 251b162250..45c24376fe 100644 --- a/docs/sources/api/docker_remote_api_v1.7.rst +++ b/docs/sources/api/docker_remote_api_v1.7.rst @@ -990,9 +990,11 @@ Build an image from Dockerfile via stdin .. sourcecode:: http HTTP/1.1 200 OK + Content-Type: application/json - {{ STREAM }} - + {"status":"Step 1..."} + {"status":"..."} + {"error":"Error...", "errorDetail":{"code": 123, "message": "Error..."}} The stream must be a tar archive compressed with one of the following algorithms: identity (no compression), gzip, bzip2, diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 20d0450a7c..8975aa7ec1 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -5,6 +5,7 @@ import ( "github.com/dotcloud/docker" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/utils" "io/ioutil" "net" "net/http" @@ -226,11 +227,14 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t, nil, true) + _, err := buildImage(ctx, t, nil, true) + if err != nil { + t.Fatal(err) + } } } -func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, useCache bool) *docker.Image { +func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, useCache bool) (*docker.Image, error) { if eng == nil { eng = NewTestEngine(t) runtime := mkRuntimeFromEngine(eng, t) @@ -262,25 +266,24 @@ func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, u } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, useCache, false) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, useCache, false, utils.NewStreamFormatter(false)) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { - t.Fatal(err) + return nil, err } - img, err := srv.ImageInspect(id) - if err != nil { - t.Fatal(err) - } - return img + return srv.ImageInspect(id) } func TestVolume(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} volume /test cmd Hello world `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if len(img.Config.Volumes) == 0 { t.Fail() @@ -293,10 +296,13 @@ func TestVolume(t *testing.T) { } func TestBuildMaintainer(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if img.Author != "dockerio" { t.Fail() @@ -304,10 +310,13 @@ func TestBuildMaintainer(t *testing.T) { } func TestBuildUser(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} user dockerio `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if img.Config.User != "dockerio" { t.Fail() @@ -315,11 +324,15 @@ func TestBuildUser(t *testing.T) { } func TestBuildEnv(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} env port 4243 `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } + hasEnv := false for _, envVar := range img.Config.Env { if envVar == "port=4243" { @@ -333,11 +346,14 @@ func TestBuildEnv(t *testing.T) { } func TestBuildCmd(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} cmd ["/bin/echo", "Hello World"] `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -350,11 +366,14 @@ func TestBuildCmd(t *testing.T) { } func TestBuildExpose(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} expose 4243 `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -362,11 +381,14 @@ func TestBuildExpose(t *testing.T) { } func TestBuildEntrypoint(t *testing.T) { - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} entrypoint ["/bin/echo"] `, nil, nil}, t, nil, true) + if err != nil { + t.Fatal(err) + } if img.Config.Entrypoint[0] != "/bin/echo" { } @@ -378,19 +400,25 @@ func TestBuildEntrypointRunCleanup(t *testing.T) { eng := NewTestEngine(t) defer nuke(mkRuntimeFromEngine(eng, t)) - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} run echo "hello" `, nil, nil}, t, eng, true) + if err != nil { + t.Fatal(err) + } - img = buildImage(testContextTemplate{` + img, err = buildImage(testContextTemplate{` from {IMAGE} run echo "hello" add foo /foo entrypoint ["/bin/echo"] `, [][2]string{{"foo", "HEYO"}}, nil}, t, eng, true) + if err != nil { + t.Fatal(err) + } if len(img.Config.Cmd) != 0 { t.Fail() @@ -407,11 +435,18 @@ func TestBuildImageWithCache(t *testing.T) { `, nil, nil} - img := buildImage(template, t, eng, true) + img, err := buildImage(template, t, eng, true) + if err != nil { + t.Fatal(err) + } + imageId := img.ID img = nil - img = buildImage(template, t, eng, true) + img, err = buildImage(template, t, eng, true) + if err != nil { + t.Fatal(err) + } if imageId != img.ID { t.Logf("Image ids should match: %s != %s", imageId, img.ID) @@ -429,11 +464,17 @@ func TestBuildImageWithoutCache(t *testing.T) { `, nil, nil} - img := buildImage(template, t, eng, true) + img, err := buildImage(template, t, eng, true) + if err != nil { + t.Fatal(err) + } imageId := img.ID img = nil - img = buildImage(template, t, eng, false) + img, err = buildImage(template, t, eng, false) + if err != nil { + t.Fatal(err) + } if imageId == img.ID { t.Logf("Image ids should not match: %s == %s", imageId, img.ID) @@ -475,7 +516,7 @@ func TestForbiddenContextPath(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, true, false) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, true, false, utils.NewStreamFormatter(false)) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { @@ -521,7 +562,7 @@ func TestBuildADDFileNotFound(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, false, true, false) + buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, false, true, false, utils.NewStreamFormatter(false)) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { @@ -539,18 +580,26 @@ func TestBuildInheritance(t *testing.T) { eng := NewTestEngine(t) defer nuke(mkRuntimeFromEngine(eng, t)) - img := buildImage(testContextTemplate{` + img, err := buildImage(testContextTemplate{` from {IMAGE} expose 4243 `, nil, nil}, t, eng, true) - img2 := buildImage(testContextTemplate{fmt.Sprintf(` + if err != nil { + t.Fatal(err) + } + + img2, _ := buildImage(testContextTemplate{fmt.Sprintf(` from %s entrypoint ["/bin/echo"] `, img.ID), nil, nil}, t, eng, true) + if err != nil { + t.Fatal(err) + } + // from child if img2.Config.Entrypoint[0] != "/bin/echo" { t.Fail() @@ -561,3 +610,23 @@ func TestBuildInheritance(t *testing.T) { t.Fail() } } + +func TestBuildFails(t *testing.T) { + _, err := buildImage(testContextTemplate{` + from {IMAGE} + run sh -c "exit 23" + `, + nil, nil}, t, nil, true) + + if err == nil { + t.Fatal("Error should not be nil") + } + + sterr, ok := err.(*utils.JSONError) + if !ok { + t.Fatalf("Error should be utils.JSONError") + } + if sterr.Code != 23 { + t.Fatalf("StatusCode %d unexpected, should be 23", sterr.Code) + } +} diff --git a/integration/commands_test.go b/integration/commands_test.go index 50cd230ce9..129a1575f8 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -905,9 +905,12 @@ run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] nil, nil, } - image := buildImage(testBuilder, t, eng, true) + image, err := buildImage(testBuilder, t, eng, true) + if err != nil { + t.Fatal(err) + } - err := mkServerFromEngine(eng, t).ContainerTag(image.ID, "test", "latest", false) + err = mkServerFromEngine(eng, t).ContainerTag(image.ID, "test", "latest", false) if err != nil { t.Fatal(err) } diff --git a/utils/utils.go b/utils/utils.go index f62aa12ff5..b6857ff4d4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1184,11 +1184,12 @@ func (graph *DependencyGraph) GenerateTraversalMap() ([][]string, error) { // An StatusError reports an unsuccessful exit by a command. type StatusError struct { - Status int + Status string + StatusCode int } func (e *StatusError) Error() string { - return fmt.Sprintf("Status: %d", e.Status) + return fmt.Sprintf("Status: %s, Code: %d", e.Status, e.StatusCode) } func quote(word string, buf *bytes.Buffer) { From e4cb83c50e554bfd8ae3e0a1f2be4f0fb661861d Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Mon, 2 Dec 2013 19:23:54 +0100 Subject: [PATCH 098/162] Bump api version and update docs --- api.go | 2 +- docs/sources/api/docker_remote_api.rst | 23 +- docs/sources/api/docker_remote_api_v1.7.rst | 4 +- docs/sources/api/docker_remote_api_v1.8.rst | 1257 +++++++++++++++++++ 4 files changed, 1277 insertions(+), 9 deletions(-) create mode 100644 docs/sources/api/docker_remote_api_v1.8.rst diff --git a/api.go b/api.go index 92ab727869..6660d396ab 100644 --- a/api.go +++ b/api.go @@ -26,7 +26,7 @@ import ( ) const ( - APIVERSION = 1.7 + APIVERSION = 1.8 DEFAULTHTTPHOST = "127.0.0.1" DEFAULTHTTPPORT = 4243 DEFAULTUNIXSOCKET = "/var/run/docker.sock" diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 5ef031ec03..4191e9a8e1 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -34,6 +34,24 @@ Calling /images//insert is the same as calling You can still call an old version of the api using /v1.0/images//insert + +v1.8 +**** + +Full Documentation +------------------ + +:doc:`docker_remote_api_v1.8` + +What's new +---------- + +.. http:post:: /build + + **New!** This endpoint now returns build status as json stream. In case + of a build error, it returns the exit status of the failed command. + + v1.7 **** @@ -138,11 +156,6 @@ What's new This URI no longer exists. The ``images -viz`` output is now generated in the client, using the ``/images/json`` data. -.. http:post:: /build - - **New!** This endpoint now returns build status as json stream. In case - of a build error, it returns the exit status of the failed command. - v1.6 **** diff --git a/docs/sources/api/docker_remote_api_v1.7.rst b/docs/sources/api/docker_remote_api_v1.7.rst index 45c24376fe..aa91210b27 100644 --- a/docs/sources/api/docker_remote_api_v1.7.rst +++ b/docs/sources/api/docker_remote_api_v1.7.rst @@ -992,9 +992,7 @@ Build an image from Dockerfile via stdin HTTP/1.1 200 OK Content-Type: application/json - {"status":"Step 1..."} - {"status":"..."} - {"error":"Error...", "errorDetail":{"code": 123, "message": "Error..."}} + {{ STREAM }} The stream must be a tar archive compressed with one of the following algorithms: identity (no compression), gzip, bzip2, diff --git a/docs/sources/api/docker_remote_api_v1.8.rst b/docs/sources/api/docker_remote_api_v1.8.rst new file mode 100644 index 0000000000..8e960739ba --- /dev/null +++ b/docs/sources/api/docker_remote_api_v1.8.rst @@ -0,0 +1,1257 @@ +:title: Remote API v1.8 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +:orphan: + +====================== +Docker Remote API v1.8 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API has replaced rcli +- The daemon listens on ``unix:///var/run/docker.sock``, but you can + :ref:`bind_docker`. +- The API tends to be REST, but for some complex commands, like + ``attach`` or ``pull``, the HTTP connection is hijacked to transport + ``stdout, stdin`` and ``stderr`` + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":[{"PrivatePort": 2222, "PublicPort": 3333, "Type": "tcp"}], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Privileged": false, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"", + "WorkingDir":"" + + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "", + "WorkingDir":"" + + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"], + "LxcConf":{"lxc.utsname":"docker"} + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a container +**************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + **Stream details**: + + When using the TTY setting is enabled in + :http:post:`/containers/create`, the stream is the raw data + from the process PTY and client's stdin. When the TTY is + disabled, then the stream is multiplexed to separate stdout + and stderr. + + The format is a **Header** and a **Payload** (frame). + + **HEADER** + + The header will contain the information on which stream write + the stream (stdout or stderr). It also contain the size of + the associated frame encoded on the last 4 bytes (uint32). + + It is encoded on the first 8 bytes like this:: + + header := [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4} + + ``STREAM_TYPE`` can be: + + - 0: stdin (will be writen on stdout) + - 1: stdout + - 2: stderr + + ``SIZE1, SIZE2, SIZE3, SIZE4`` are the 4 bytes of the uint32 size encoded as big endian. + + **PAYLOAD** + + The payload is the raw stream. + + **IMPLEMENTATION** + + The simplest way to implement the Attach protocol is the following: + + 1) Read 8 bytes + 2) chose stdout or stderr depending on the first byte + 3) Extract the frame size from the last 4 byets + 4) Read the extracted size and output it on the correct output + 5) Goto 1) + + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Copy files or folders from a container +************************************** + +.. http:post:: /containers/(id)/copy + + Copy files or folders of container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/4fa6e0f0c678/copy HTTP/1.1 + Content-Type: application/json + + { + "Resource":"test.txt" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/json + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "RepoTag": [ + "ubuntu:12.04", + "ubuntu:precise", + "ubuntu:latest" + ], + "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c", + "Created": 1365714795, + "Size": 131506275, + "VirtualSize": 131506275 + }, + { + "RepoTag": [ + "ubuntu:12.10", + "ubuntu:quantal" + ], + "ParentId": "27cf784147099545", + "Id": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "Created": 1364102658, + "Size": 24653, + "VirtualSize": 180116135 + } + ] + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + When using this endpoint to pull an image from the registry, + the ``X-Registry-Auth`` header can be used to include a + base64-encoded AuthConfig object. + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :reqheader X-Registry-Auth: base64-encoded AuthConfig object + :statuscode 200: no error + :statuscode 500: server error + + + +Insert a file in an image +************************* + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"", + "WorkingDir":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :reqheader X-Registry-Auth: include a base64-encoded AuthConfig object. + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index. + + .. note:: + + The response keys have changed from API v1.6 to reflect the JSON + sent by the registry server to the docker daemon's request. + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "description": "", + "is_official": false, + "is_trusted": false, + "name": "wma55/u1210sshd", + "star_count": 0 + }, + { + "description": "", + "is_official": false, + "is_trusted": false, + "name": "jdswinbank/sshd", + "star_count": 0 + }, + { + "description": "", + "is_official": false, + "is_trusted": false, + "name": "vgauthier/sshd", + "star_count": 0 + } + ... + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Step 1..."} + {"status":"..."} + {"error":"Error...", "errorDetail":{"code": 123, "message": "Error..."}} + + + The stream must be a tar archive compressed with one of the + following algorithms: identity (no compression), gzip, bzip2, + xz. + + The archive must include a file called ``Dockerfile`` at its + root. It may include any number of other files, which will be + accessible in the build context (See the :ref:`ADD build command + `). + + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success + :query q: suppress verbose build output + :query nocache: do not use the cache when building the image + :reqheader Content-type: should be set to ``"application/tar"``. + :statuscode 200: no error + :statuscode 500: server error + + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com", + "serveraddress":"https://index.docker.io/v1/" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false, + "IPv4Forwarding":true + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Monitor Docker's events +*********************** + +.. http:get:: /events + + Get events from docker, either in real time via streaming, or via polling (using `since`) + + **Example request**: + + .. sourcecode:: http + + POST /events?since=1374067924 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"create","id":"dfdf82bd3881","from":"base:latest","time":1374067924} + {"status":"start","id":"dfdf82bd3881","from":"base:latest","time":1374067924} + {"status":"stop","id":"dfdf82bd3881","from":"base:latest","time":1374067966} + {"status":"destroy","id":"dfdf82bd3881","from":"base:latest","time":1374067970} + + :query since: timestamp used for polling + :statuscode 200: no error + :statuscode 500: server error + +Get a tarball containing all images and tags in a repository +************************************************************ + +.. http:get:: /images/(name)/get + + Get a tarball containing all images and metadata for the repository specified by ``name``. + + **Example request** + + .. sourcecode:: http + + GET /images/ubuntu/get + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/x-tar + + Binary data stream + :statuscode 200: no error + :statuscode 500: server error + +Load a tarball with a set of images and tags into docker +******************************************************** + +.. http:post:: /images/load + + Load a set of images and tags into the docker repository. + + **Example request** + + .. sourcecode:: http + + POST /images/load + + Tarball in body + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 500: server error + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + +.. code-block:: bash + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From 70b586702cb1715d4ff4dd7ffe356285df73b18e Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Date: Fri, 29 Nov 2013 01:54:41 -0500 Subject: [PATCH 099/162] Add contrb/zfs/ and add self as maintainer of ZFS storage driver. --- contrib/zfs/MAINTAINERS | 1 + contrib/zfs/README.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 contrib/zfs/MAINTAINERS create mode 100644 contrib/zfs/README.md diff --git a/contrib/zfs/MAINTAINERS b/contrib/zfs/MAINTAINERS new file mode 100644 index 0000000000..90bc6e3d60 --- /dev/null +++ b/contrib/zfs/MAINTAINERS @@ -0,0 +1 @@ +Gurjeet Singh (gurjeet.singh.im) diff --git a/contrib/zfs/README.md b/contrib/zfs/README.md new file mode 100644 index 0000000000..7789180cad --- /dev/null +++ b/contrib/zfs/README.md @@ -0,0 +1,22 @@ +# ZFS Stroage Driver + +This is a placeholder to declare the presence and status of ZFS storage driver +for containers. + +The current development is done in Gurjeet Singh's fork of Docker, under the +branch named [zfs_driver]. + +[zfs_driver]: https://github.com/gurjeet/docker/tree/zfs_driver + + +# Status + +Pre-alpha + +The code is under development. Contributions in the form of suggestions, +code-reviews, and patches are welcome. + +Please send the communication to gurjeet@singh.im and CC at least one Docker +mailing list. + + From 9837ad8e9b96b118e8d004e773a05ff9dd208b48 Mon Sep 17 00:00:00 2001 From: Dustin Sallings Date: Wed, 27 Nov 2013 15:07:15 -0800 Subject: [PATCH 100/162] Add -format to 'docker inspect' This makes it a lot easier to script with docker instances as one can ask for details about running instances more easily without having to have additional JSON processing tools installed. dotcloud/docker#734 --- commands.go | 50 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/commands.go b/commands.go index e24b7aeece..ec51983ca0 100644 --- a/commands.go +++ b/commands.go @@ -31,6 +31,7 @@ import ( "strings" "syscall" "text/tabwriter" + "text/template" "time" ) @@ -632,6 +633,7 @@ func (cli *DockerCli) CmdStart(args ...string) error { func (cli *DockerCli) CmdInspect(args ...string) error { cmd := cli.Subcmd("inspect", "CONTAINER|IMAGE [CONTAINER|IMAGE...]", "Return low-level information on a container/image") + tmplStr := cmd.String("format", "", "Format the output using the given go template.") if err := cmd.Parse(args); err != nil { return nil } @@ -640,10 +642,21 @@ func (cli *DockerCli) CmdInspect(args ...string) error { return nil } + var tmpl *template.Template + if *tmplStr != "" { + var err error + if tmpl, err = template.New("").Parse(*tmplStr); err != nil { + fmt.Fprintf(cli.err, "Template parsing error: %v\n", err) + return &utils.StatusError{StatusCode: 64, + Status: "Template parsing error: " + err.Error()} + } + } + indented := new(bytes.Buffer) + indented.WriteByte('[') status := 0 - for _, name := range args { + for _, name := range cmd.Args() { obj, _, err := cli.call("GET", "/containers/"+name+"/json", nil) if err != nil { obj, _, err = cli.call("GET", "/images/"+name+"/json", nil) @@ -658,23 +671,40 @@ func (cli *DockerCli) CmdInspect(args ...string) error { } } - if err = json.Indent(indented, obj, "", " "); err != nil { - fmt.Fprintf(cli.err, "%s\n", err) - status = 1 - continue + if tmpl == nil { + if err = json.Indent(indented, obj, "", " "); err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + status = 1 + continue + } + } else { + // Has template, will render + var value interface{} + if err := json.Unmarshal(obj, &value); err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + status = 1 + continue + } + if err := tmpl.Execute(cli.out, value); err != nil { + return err + } + cli.out.Write([]byte{'\n'}) } indented.WriteString(",") } - if indented.Len() > 0 { + if indented.Len() > 1 { // Remove trailing ',' indented.Truncate(indented.Len() - 1) } - fmt.Fprintf(cli.out, "[") - if _, err := io.Copy(cli.out, indented); err != nil { - return err + indented.WriteByte(']') + + if tmpl == nil { + if _, err := io.Copy(cli.out, indented); err != nil { + return err + } } - fmt.Fprintf(cli.out, "]") + if status != 0 { return &utils.StatusError{StatusCode: status} } From 1d503be4660667a27b42f7bf0c089404cabd2c9c Mon Sep 17 00:00:00 2001 From: Dustin Sallings Date: Fri, 29 Nov 2013 16:12:10 -0800 Subject: [PATCH 101/162] Use inspect format to get IP address for psql example --- docs/sources/examples/postgresql_service.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/examples/postgresql_service.rst b/docs/sources/examples/postgresql_service.rst index 82ca8b59ca..e142f65899 100644 --- a/docs/sources/examples/postgresql_service.rst +++ b/docs/sources/examples/postgresql_service.rst @@ -127,7 +127,7 @@ on the machine. For ubuntu, use something like .. code-block:: bash - CONTAINER_IP=$(sudo docker inspect $CONTAINER | grep IPAddress | awk '{ print $2 }' | tr -d ',"') + CONTAINER_IP=$(sudo docker inspect -format='{{.NetworkSettings.IPAddress}}' $CONTAINER) psql -h $CONTAINER_IP -p 5432 -d docker -U docker -W As before, create roles or databases if needed. From 4ad3dfb05f1932793cae5c8047a89660747b95b4 Mon Sep 17 00:00:00 2001 From: Dustin Sallings Date: Fri, 29 Nov 2013 16:36:50 -0800 Subject: [PATCH 102/162] CLI docs and examples of format --- docs/sources/commandline/cli.rst | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index a6f15df8a6..d7bb03b954 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -659,6 +659,54 @@ Insert file from github Return low-level information on a container + -format="": template to output results + +By default, this will render all results in a JSON array. If a format +is specified, the given template will be executed for each result. + +Go's `text/template ` package +describes all the details of the format. + +Examples +~~~~~~~~ + +Get an instance's IP Address +............................ + +For the most part, you can pick out any field from the JSON in a +fairly straightforward manner. + +.. code-block:: bash + + docker inspect -format='{{.NetworkSettings.IPAddress}}' $INSTANCE_ID + +List All Port Bindings +...................... + +One can loop over arrays and maps in the results to produce simple +text output: + +.. code-block:: bash + + docker inspect -format='{{range $p, $conf := .NetworkSettings.Ports}} {{$p}} -> {{(index $conf 0).HostPort}} {{end}}' $INSTANCE_ID + +Find a Specific Port Mapping +............................ + +.. code-block:: bash + +The ``.Field`` syntax doesn't work when the field name begins with a +number, but the template language's ``index`` function does. The +``.NetworkSettings.Ports`` section contains a map of the internal port +mappings to a list of external address/port objects, so to grab just +the numeric public port, you use ``index`` to find the specific port +map, and then ``index`` 0 contains first object inside of that. Then +we ask for the ``HostPort`` field to get the public address. + +.. code-block:: bash + + docker inspect -format='{{(index (index .NetworkSettings.Ports "8787/tcp") 0).HostPort}}' $INSTANCE_ID + .. _cli_kill: ``kill`` From de4429f70d51056c3ec32072f3f18898e36171b3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 11:43:41 -0800 Subject: [PATCH 103/162] Do not format at each write but use a Writer instead (build) --- api.go | 15 ++++++++-- buildfile.go | 83 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/api.go b/api.go index 6660d396ab..da16044410 100644 --- a/api.go +++ b/api.go @@ -960,11 +960,20 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ return err } - if version > 1.6 { + if version >= 1.8 { w.Header().Set("Content-Type", "application/json") } - sf := utils.NewStreamFormatter(version > 1.6) - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache, rm, sf) + sf := utils.NewStreamFormatter(version >= 1.8) + b := NewBuildFile(srv, + &StdoutFormater{ + Writer: utils.NewWriteFlusher(w), + StreamFormatter: sf, + }, + &StderrFormater{ + Writer: utils.NewWriteFlusher(w), + StreamFormatter: sf, + }, + !suppressOutput, !noCache, rm, utils.NewWriteFlusher(w), sf) id, err := b.Build(context) if err != nil { if sf.Used() { diff --git a/buildfile.go b/buildfile.go index 92b3297843..e8f178fbbd 100644 --- a/buildfile.go +++ b/buildfile.go @@ -36,15 +36,19 @@ type buildFile struct { tmpContainers map[string]struct{} tmpImages map[string]struct{} - out io.Writer - sf *utils.StreamFormatter + outStream io.Writer + errStream io.Writer + + // Deprecated, original writer used for ImagePull. To be removed. + outOld io.Writer + sf *utils.StreamFormatter } func (b *buildFile) clearTmp(containers map[string]struct{}) { for c := range containers { tmp := b.runtime.Get(c) b.runtime.Destroy(tmp) - fmt.Fprintf(b.out, "Removing intermediate container %s\n", utils.TruncateID(c)) + fmt.Fprintf(b.outStream, "Removing intermediate container %s\n", utils.TruncateID(c)) } } @@ -53,7 +57,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.out, b.sf, nil, nil, true); err != nil { + if err := b.srv.ImagePull(remote, tag, b.outOld, b.sf, nil, nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) @@ -101,11 +105,7 @@ func (b *buildFile) CmdRun(args string) error { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return err } else if cache != nil { - if b.sf.Used() { - b.out.Write(b.sf.FormatStatus("", " ---> Using cache")) - } else { - fmt.Fprintf(b.out, " ---> Using cache\n") - } + fmt.Fprintf(b.outStream, " ---> Using cache\n") utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return nil @@ -369,6 +369,34 @@ func (b *buildFile) CmdAdd(args string) error { return nil } +type StdoutFormater struct { + io.Writer + *utils.StreamFormatter +} + +func (sf *StdoutFormater) Write(buf []byte) (int, error) { + formattedBuf := sf.StreamFormatter.FormatStatus("", "%s", string(buf)) + n, err := sf.Writer.Write(formattedBuf) + if n != len(formattedBuf) { + return n, io.ErrShortWrite + } + return len(buf), err +} + +type StderrFormater struct { + io.Writer + *utils.StreamFormatter +} + +func (sf *StderrFormater) Write(buf []byte) (int, error) { + formattedBuf := sf.StreamFormatter.FormatStatus("", "%s", "\033[91m"+string(buf)+"\033[0m") + n, err := sf.Writer.Write(formattedBuf) + if n != len(formattedBuf) { + return n, io.ErrShortWrite + } + return len(buf), err +} + func (b *buildFile) run() (string, error) { if b.image == "" { return "", fmt.Errorf("Please provide a source image with `from` prior to run") @@ -381,11 +409,8 @@ func (b *buildFile) run() (string, error) { return "", err } b.tmpContainers[c.ID] = struct{}{} - if b.sf.Used() { - b.out.Write(b.sf.FormatStatus("", " ---> Running in %s", utils.TruncateID(c.ID))) - } else { - fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(c.ID)) - } + fmt.Fprintf(b.outStream, " ---> Running in %s\n", utils.TruncateID(c.ID)) + // override the entry point that may have been picked up from the base image c.Path = b.config.Cmd[0] c.Args = b.config.Cmd[1:] @@ -394,7 +419,7 @@ func (b *buildFile) run() (string, error) { if b.verbose { errCh = utils.Go(func() error { - return <-c.Attach(nil, nil, b.out, b.out) + return <-c.Attach(nil, nil, b.outStream, b.errStream) }) } @@ -436,11 +461,7 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return err } else if cache != nil { - if b.sf.Used() { - b.out.Write(b.sf.FormatStatus("", " ---> Using cache")) - } else { - fmt.Fprintf(b.out, " ---> Using cache\n") - } + fmt.Fprintf(b.outStream, " ---> Using cache\n") utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return nil @@ -454,14 +475,10 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { return err } for _, warning := range warnings { - fmt.Fprintf(b.out, " ---> [Warning] %s\n", warning) + fmt.Fprintf(b.outStream, " ---> [Warning] %s\n", warning) } b.tmpContainers[container.ID] = struct{}{} - if b.sf.Used() { - b.out.Write(b.sf.FormatStatus("", " ---> Running in %s", utils.TruncateID(container.ID))) - } else { - fmt.Fprintf(b.out, " ---> Running in %s\n", utils.TruncateID(container.ID)) - } + fmt.Fprintf(b.outStream, " ---> Running in %s\n", utils.TruncateID(container.ID)) id = container.ID if err := container.EnsureMounted(); err != nil { return err @@ -527,22 +544,22 @@ func (b *buildFile) Build(context io.Reader) (string, error) { method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) if !exists { - b.out.Write(b.sf.FormatStatus("", "# Skipping unknown instruction %s", strings.ToUpper(instruction))) + fmt.Fprintf(b.errStream, "# Skipping unknown instruction %s\n", strings.ToUpper(instruction)) continue } stepN += 1 - b.out.Write(b.sf.FormatStatus("", "Step %d : %s %s", stepN, strings.ToUpper(instruction), arguments)) + fmt.Fprintf(b.outStream, "Step %d : %s %s\n", stepN, strings.ToUpper(instruction), arguments) ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() if ret != nil { return "", ret.(error) } - b.out.Write(b.sf.FormatStatus("", " ---> %s", utils.TruncateID(b.image))) + fmt.Fprintf(b.outStream, " ---> %s\n", utils.TruncateID(b.image)) } if b.image != "" { - b.out.Write(b.sf.FormatStatus("", "Successfully built %s", utils.TruncateID(b.image))) + fmt.Fprintf(b.outStream, "Successfully built %s\n", utils.TruncateID(b.image)) if b.rm { b.clearTmp(b.tmpContainers) } @@ -551,17 +568,19 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occurred during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache, rm bool, sf *utils.StreamFormatter) BuildFile { +func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeCache, rm bool, outOld io.Writer, sf *utils.StreamFormatter) BuildFile { return &buildFile{ runtime: srv.runtime, srv: srv, config: &Config{}, - out: out, + outStream: outStream, + errStream: errStream, tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, utilizeCache: utilizeCache, rm: rm, sf: sf, + outOld: outOld, } } From 6ea3b9651b3793e31a320926472ff23383a7b915 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 11:47:13 -0800 Subject: [PATCH 104/162] Fix displayJson behavior (dont add newline) --- commands.go | 2 ++ utils/jsonmessage.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index e24b7aeece..19e7110e9d 100644 --- a/commands.go +++ b/commands.go @@ -228,6 +228,8 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if context != nil { headers.Set("Content-Type", "application/tar") } + // Temporary hack to fix displayJSON behavior + cli.isTerminal = false err = cli.stream("POST", fmt.Sprintf("/build?%s", v.Encode()), body, cli.out, headers) if jerr, ok := err.(*utils.JSONError); ok { return &utils.StatusError{Status: jerr.Message, StatusCode: jerr.Code} diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index a0aeba2e7a..be74033f7f 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -63,7 +63,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { if isTerminal { // [2K = erase entire current line fmt.Fprintf(out, "%c[2K\r", 27) - endl = "\r" + endl = "\r\n" } if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) @@ -79,7 +79,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { } else if jm.ProgressMessage != "" { //deprecated fmt.Fprintf(out, "%s %s%s", jm.Status, jm.ProgressMessage, endl) } else { - fmt.Fprintf(out, "%s%s\n", jm.Status, endl) + fmt.Fprintf(out, "%s%s", jm.Status, endl) } return nil } From 5cd09dc1158c62754641dce9c59a735c41e59722 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 11:49:11 -0800 Subject: [PATCH 105/162] small reformatting jsonmessage --- utils/jsonmessage.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index be74033f7f..809524cd28 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -59,7 +59,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { } return jm.Error } - endl := "" + var endl string if isTerminal { // [2K = erase entire current line fmt.Fprintf(out, "%c[2K\r", 27) @@ -85,14 +85,17 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { } func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) error { - dec := json.NewDecoder(in) - ids := make(map[string]int) - diff := 0 + var ( + dec = json.NewDecoder(in) + ids = make(map[string]int) + diff = 0 + ) for { - jm := JSONMessage{} - if err := dec.Decode(&jm); err == io.EOF { - break - } else if err != nil { + var jm JSONMessage + if err := dec.Decode(&jm); err != nil { + if err == io.EOF { + break + } return err } if (jm.Progress != nil || jm.ProgressMessage != "") && jm.ID != "" { From 98ed1dc433e423b93ae0619c5a2c474ccec1454d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 12:51:37 -0800 Subject: [PATCH 106/162] Fix unit test with new buildfile prototype --- integration/buildfile_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index 8975aa7ec1..242bf9f412 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -266,7 +266,7 @@ func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, u } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, useCache, false, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, ioutil.Discard, utils.NewStreamFormatter(false)) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { return nil, err @@ -516,7 +516,7 @@ func TestForbiddenContextPath(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, true, false, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false)) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { @@ -562,7 +562,7 @@ func TestBuildADDFileNotFound(t *testing.T) { } dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, false, true, false, utils.NewStreamFormatter(false)) + buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false)) _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err == nil { From 3cd9b2aadfd16736032ee569ba263fce53286347 Mon Sep 17 00:00:00 2001 From: JP Date: Fri, 22 Nov 2013 12:55:27 -0500 Subject: [PATCH 107/162] Fixes #2820 --- config.go | 4 ++-- docker/docker.go | 5 +++-- opts.go | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index eb00e41fa1..51d435b841 100644 --- a/config.go +++ b/config.go @@ -27,8 +27,8 @@ func ConfigFromJob(job *engine.Job) *DaemonConfig { config.Root = job.Getenv("Root") config.AutoRestart = job.GetenvBool("AutoRestart") config.EnableCors = job.GetenvBool("EnableCors") - if dns := job.Getenv("Dns"); dns != "" { - config.Dns = []string{dns} + if dns := job.GetenvList("Dns"); dns != nil { + config.Dns = dns } config.EnableIptables = job.GetenvBool("EnableIptables") if br := job.Getenv("BridgeIface"); br != "" { diff --git a/docker/docker.go b/docker/docker.go index ca829f3564..6e243f157f 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -33,13 +33,14 @@ func main() { pidfile = flag.String("p", "/var/run/docker.pid", "Path to use for daemon PID file") flRoot = flag.String("g", "/var/lib/docker", "Path to use as the root of the docker runtime") flEnableCors = flag.Bool("api-enable-cors", false, "Enable CORS headers in the remote API") - flDns = flag.String("dns", "", "Force docker to use specific DNS servers") + flDns = docker.NewListOpts(docker.ValidateIp4Address) flEnableIptables = flag.Bool("iptables", true, "Disable docker's addition of iptables rules") flDefaultIp = flag.String("ip", "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm = flag.Bool("icc", true, "Enable inter-container communication") flGraphDriver = flag.String("s", "", "Force the docker runtime to use a specific storage driver") flHosts = docker.NewListOpts(docker.ValidateHost) ) + flag.Var(&flDns, "dns", "Force docker to use specific DNS servers") flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") flag.Parse() @@ -73,7 +74,7 @@ func main() { job.Setenv("Root", *flRoot) job.SetenvBool("AutoRestart", *flAutoRestart) job.SetenvBool("EnableCors", *flEnableCors) - job.Setenv("Dns", *flDns) + job.SetenvList("Dns", flDns.GetAll()) job.SetenvBool("EnableIptables", *flEnableIptables) job.Setenv("BridgeIface", *bridgeName) job.Setenv("DefaultIp", *flDefaultIp) diff --git a/opts.go b/opts.go index 80785a5161..ea5163e619 100644 --- a/opts.go +++ b/opts.go @@ -5,6 +5,7 @@ import ( "github.com/dotcloud/docker/utils" "os" "path/filepath" + "regexp" "strings" ) @@ -134,3 +135,12 @@ func ValidateHost(val string) (string, error) { } return host, nil } + +func ValidateIp4Address(val string) (string, error) { + re := regexp.MustCompile(`^(([0-9]+\.){3}([0-9]+))\s*$`) + var ns = re.FindSubmatch([]byte(val)) + if len(ns) > 0 { + return string(ns[1]), nil + } + return "", fmt.Errorf("%s is not an ip4 address", val) +} From 0017c68f4a56f9250b53e4f8efdf3513b051c1a3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 13:52:27 -0800 Subject: [PATCH 108/162] Remove trailing whitespace --- hack/make/test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/make/test b/hack/make/test index eab8c18c84..8f4dac0551 100644 --- a/hack/make/test +++ b/hack/make/test @@ -7,25 +7,25 @@ set -e # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. -# +# # TESTFLAGS='-run ^TestBuild$' ./hack/make.sh test # bundle_test() { { date - + TESTS_FAILED=() for test_dir in $(find_test_dirs); do echo - + if ! ( set -x cd $test_dir - + # Install packages that are dependencies of the tests. # Note: Does not run the tests. go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS - + # Run the tests with the optional $TESTFLAGS. go test -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS ); then @@ -33,7 +33,7 @@ bundle_test() { sleep 1 # give it a second, so observers watching can take note fi done - + # if some tests fail, we want the bundlescript to fail, but we want to # try running ALL the tests first, hence TESTS_FAILED if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then From 9d62dc1a0837e37ce5da01c278ab20f7d32c3774 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 23 Oct 2013 11:22:41 -0600 Subject: [PATCH 109/162] Remove mkimage-centos.sh in favor of a new rinse-based script, since febootstrap is fragile and picky --- contrib/mkimage-centos.sh | 15 ----- contrib/mkimage-rinse.sh | 112 ++++++++++++++++++++++++++++++++ docs/sources/use/baseimages.rst | 6 +- 3 files changed, 115 insertions(+), 18 deletions(-) delete mode 100755 contrib/mkimage-centos.sh create mode 100755 contrib/mkimage-rinse.sh diff --git a/contrib/mkimage-centos.sh b/contrib/mkimage-centos.sh deleted file mode 100755 index a3c3253f97..0000000000 --- a/contrib/mkimage-centos.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Create a CentOS base image for Docker -# From unclejack https://github.com/dotcloud/docker/issues/290 -set -e - -MIRROR_URL="http://centos.netnitco.net/6.4/os/x86_64/" -MIRROR_URL_UPDATES="http://centos.netnitco.net/6.4/updates/x86_64/" - -yum install -y febootstrap xz - -febootstrap -i bash -i coreutils -i tar -i bzip2 -i gzip -i vim-minimal -i wget -i patch -i diffutils -i iproute -i yum centos centos64 $MIRROR_URL -u $MIRROR_URL_UPDATES -touch centos64/etc/resolv.conf -touch centos64/sbin/init - -tar --numeric-owner -Jcpf centos-64.tar.xz -C centos64 . diff --git a/contrib/mkimage-rinse.sh b/contrib/mkimage-rinse.sh new file mode 100755 index 0000000000..ff8f173f98 --- /dev/null +++ b/contrib/mkimage-rinse.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -e + +repo="$1" +distro="$2" +mirror="$3" + +if [ ! "$repo" ] || [ ! "$distro" ]; then + self="$(basename $0)" + echo >&2 "usage: $self repo distro [mirror]" + echo >&2 + echo >&2 " ie: $self username/centos centos-5" + echo >&2 " $self username/centos centos-6" + echo >&2 + echo >&2 " ie: $self username/slc slc-5" + echo >&2 " $self username/slc slc-6" + echo >&2 + echo >&2 " ie: $self username/centos centos-5 http://vault.centos.org/5.8/os/x86_64/CentOS/" + echo >&2 " $self username/centos centos-6 http://vault.centos.org/6.3/os/x86_64/Packages/" + echo >&2 + echo >&2 'See /etc/rinse for supported values of "distro" and for examples of' + echo >&2 ' expected values of "mirror".' + echo >&2 + echo >&2 'This script is tested to work with the original upstream version of rinse,' + echo >&2 ' found at http://www.steve.org.uk/Software/rinse/ and also in Debian at' + echo >&2 ' http://packages.debian.org/wheezy/rinse -- as always, YMMV.' + echo >&2 + exit 1 +fi + +target="/tmp/docker-rootfs-rinse-$distro-$$-$RANDOM" + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" +returnTo="$(pwd -P)" + +rinseArgs=( --arch amd64 --distribution "$distro" --directory "$target" ) +if [ "$mirror" ]; then + rinseArgs+=( --mirror "$mirror" ) +fi + +set -x + +mkdir -p "$target" + +sudo rinse "${rinseArgs[@]}" + +cd "$target" + +# rinse fails a little at setting up /dev, so we'll just wipe it out and create our own +sudo rm -rf dev +sudo mkdir -m 755 dev +( + cd dev + sudo ln -sf /proc/self/fd ./ + sudo mkdir -m 755 pts + sudo mkdir -m 1777 shm + sudo mknod -m 600 console c 5 1 + sudo mknod -m 600 initctl p + sudo mknod -m 666 full c 1 7 + sudo mknod -m 666 null c 1 3 + sudo mknod -m 666 ptmx c 5 2 + sudo mknod -m 666 random c 1 8 + sudo mknod -m 666 tty c 5 0 + sudo mknod -m 666 tty0 c 4 0 + sudo mknod -m 666 urandom c 1 9 + sudo mknod -m 666 zero c 1 5 +) + +# effectively: febootstrap-minimize --keep-zoneinfo --keep-rpmdb --keep-services "$target" +# locales +sudo rm -rf usr/{{lib,share}/locale,{lib,lib64}/gconv,bin/localedef,sbin/build-locale-archive} +# docs +sudo rm -rf usr/share/{man,doc,info,gnome/help} +# cracklib +sudo rm -rf usr/share/cracklib +# i18n +sudo rm -rf usr/share/i18n +# yum cache +sudo rm -rf var/cache/yum +sudo mkdir -p --mode=0755 var/cache/yum +# sln +sudo rm -rf sbin/sln +# ldconfig +#sudo rm -rf sbin/ldconfig +sudo rm -rf etc/ld.so.cache var/cache/ldconfig +sudo mkdir -p --mode=0755 var/cache/ldconfig + +# allow networking init scripts inside the container to work without extra steps +echo 'NETWORKING=yes' | sudo tee etc/sysconfig/network > /dev/null + +# to restore locales later: +# yum reinstall glibc-common + +version= +if [ -r etc/redhat-release ]; then + version="$(sed -E 's/^[^0-9.]*([0-9.]+).*$/\1/' etc/redhat-release)" +elif [ -r etc/SuSE-release ]; then + version="$(awk '/^VERSION/ { print $3 }' etc/SuSE-release)" +fi + +if [ -z "$version" ]; then + echo >&2 "warning: cannot autodetect OS version, using $distro as tag" + sleep 20 + version="$distro" +fi + +sudo tar --numeric-owner -c . | docker import - $repo:$version + +docker run -i -t $repo:$version echo success + +cd "$returnTo" +sudo rm -rf "$target" diff --git a/docs/sources/use/baseimages.rst b/docs/sources/use/baseimages.rst index 4785cc5298..f1f5d106e0 100644 --- a/docs/sources/use/baseimages.rst +++ b/docs/sources/use/baseimages.rst @@ -37,7 +37,7 @@ There are more example scripts for creating base images in the Docker Github Repo: * `BusyBox `_ -* `CentOS - `_ -* `Debian/Ubuntu +* `CentOS / Scientific Linux CERN (SLC) + `_ +* `Debian / Ubuntu `_ From e4aba11e80561d06e457453c58def970518b691c Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 2 Dec 2013 11:43:56 -0600 Subject: [PATCH 110/162] add env variable to disable kernel version warning Allow the user to set DOCKER_NOWARN_KERNEL_VERSION=1 to disable the warning for RHEL 6.5 and other distributions that don't exhibit the panics described in https://github.com/dotcloud/docker/issues/407. --- engine/engine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/engine.go b/engine/engine.go index 5a411e8cc2..623af81bd1 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -70,7 +70,9 @@ func New(root string) (*Engine, error) { log.Printf("WARNING: %s\n", err) } else { if utils.CompareKernelVersion(k, &utils.KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { - log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) + if os.Getenv("DOCKER_NOWARN_KERNEL_VERSION") == "" { + log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) + } } } if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { From 5a9cf7e75442ce1c76876da717b8c2f321fc04ae Mon Sep 17 00:00:00 2001 From: JP Date: Mon, 2 Dec 2013 17:33:33 -0500 Subject: [PATCH 111/162] Add unit test for ValidateIp4Address --- opts_unit_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 opts_unit_test.go diff --git a/opts_unit_test.go b/opts_unit_test.go new file mode 100644 index 0000000000..67b061771b --- /dev/null +++ b/opts_unit_test.go @@ -0,0 +1,24 @@ +package docker + +import ( + "testing" +) + +func TestValidateIP4(t *testing.T) { + if ret, err := ValidateIp4Address(`1.2.3.4`); err != nil || ret == "" { + t.Fatalf("ValidateIp4Address(`1.2.3.4`) got %s %s", ret, err) + } + + if ret, err := ValidateIp4Address(`127.0.0.1`); err != nil || ret == "" { + t.Fatalf("ValidateIp4Address(`127.0.0.1`) got %s %s", ret, err) + } + + if ret, err := ValidateIp4Address(`127`); err == nil || ret != "" { + t.Fatalf("ValidateIp4Address(`127`) got %s %s", ret, err) + } + + if ret, err := ValidateIp4Address(`random invalid string`); err == nil || ret != "" { + t.Fatalf("ValidateIp4Address(`random invalid string`) got %s %s", ret, err) + } + +} From 3ac76cfeffafcfdc3688fda58cb29d9bb7f6149b Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 2 Dec 2013 15:48:26 -0700 Subject: [PATCH 112/162] Update bundlescript shebangs to be bash, reflecting how they're actually invoked --- hack/make/binary | 2 +- hack/make/dynbinary | 2 +- hack/make/dyntest | 2 +- hack/make/test | 2 +- hack/make/tgz | 2 +- hack/make/ubuntu | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/make/binary b/hack/make/binary index 6363c2ab09..93e99fee8f 100644 --- a/hack/make/binary +++ b/hack/make/binary @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST=$1 diff --git a/hack/make/dynbinary b/hack/make/dynbinary index e630c94f53..96ce482674 100644 --- a/hack/make/dynbinary +++ b/hack/make/dynbinary @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST=$1 diff --git a/hack/make/dyntest b/hack/make/dyntest index a4ab2e456b..61f03ada1d 100644 --- a/hack/make/dyntest +++ b/hack/make/dyntest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST=$1 INIT=$DEST/../dynbinary/dockerinit-$VERSION diff --git a/hack/make/test b/hack/make/test index eab8c18c84..b6dd7f0916 100644 --- a/hack/make/test +++ b/hack/make/test @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST=$1 diff --git a/hack/make/tgz b/hack/make/tgz index 80cdb69eed..86a7a9827d 100644 --- a/hack/make/tgz +++ b/hack/make/tgz @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST="$1" BINARY="$DEST/../binary/docker-$VERSION" diff --git a/hack/make/ubuntu b/hack/make/ubuntu index d4b9fd0b30..578f254558 100644 --- a/hack/make/ubuntu +++ b/hack/make/ubuntu @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash DEST=$1 From 829b118dd8fbfb9ce1585c15699cf6221c328802 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 2 Dec 2013 13:52:48 -0800 Subject: [PATCH 113/162] Add some color in order to emphasis the test FAILURE --- hack/make/test | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hack/make/test b/hack/make/test index 8f4dac0551..8e1103bacc 100644 --- a/hack/make/test +++ b/hack/make/test @@ -4,6 +4,10 @@ DEST=$1 set -e +TEXTRESET=$'\033[0m' # reset the foreground colour +RED=$'\033[31m' +GREEN=$'\033[32m' + # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, eg. @@ -30,6 +34,9 @@ bundle_test() { go test -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS ); then TESTS_FAILED+=("$test_dir") + echo + echo "${RED}Test Failed: $test_dir${TEXTRESET}" + echo sleep 1 # give it a second, so observers watching can take note fi done @@ -38,8 +45,12 @@ bundle_test() { # try running ALL the tests first, hence TESTS_FAILED if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then echo - echo "Test failures in: ${TESTS_FAILED[@]}" + echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" false + else + echo + echo "${GREEN}Test success${TEXTRESET}" + true fi } 2>&1 | tee $DEST/test.log } From c9432cf51aea9351cc88791036b986e6662a49b8 Mon Sep 17 00:00:00 2001 From: John Feminella Date: Sat, 23 Nov 2013 15:30:29 -0500 Subject: [PATCH 114/162] Reformats source text to proper widths --- docs/sources/installation/amazon.rst | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/sources/installation/amazon.rst b/docs/sources/installation/amazon.rst index 0c36dbfc33..5e92fa8df3 100644 --- a/docs/sources/installation/amazon.rst +++ b/docs/sources/installation/amazon.rst @@ -22,20 +22,27 @@ Amazon QuickStart 1. **Choose an image:** - * Launch the `Create Instance Wizard` menu on your AWS Console - * Select "Community AMIs" option and serch for ``amd64 precise`` (click enter to search) - * If you choose a EBS enabled AMI you will be able to launch a `t1.micro` instance (more info on `pricing` ) - * When you click select you'll be taken to the instance setup, and you're one click away from having your Ubuntu VM up and running. + * Launch the `Create Instance Wizard` + menu on + your AWS Console + + * Select "Community AMIs" option and serch for ``amd64 precise`` (click enter + to search) + + * If you choose a EBS enabled AMI you will be able to launch a `t1.micro` + instance (more info on `pricing` ) + + * When you click select you'll be taken to the instance setup, and you're one + click away from having your Ubuntu VM up and running. 2. **Tell CloudInit to install Docker:** - * Enter ``#include https://get.docker.io`` into the instance *User - Data*. `CloudInit `_ - is part of the Ubuntu image you chose and it bootstraps from this - *User Data*. + * Enter ``#include https://get.docker.io`` into the instance *User Data*. + `CloudInit `_ is part of the + Ubuntu image you chose and it bootstraps from this *User Data*. -3. After a few more standard choices where defaults are probably ok, your - AWS Ubuntu instance with Docker should be running! +3. After a few more standard choices where defaults are probably ok, your AWS + Ubuntu instance with Docker should be running! **If this is your first AWS instance, you may need to set up your Security Group to allow SSH.** By default all incoming ports to your From 6c70d23e0d850b31fce0e658ad9ddd7a167bcdcb Mon Sep 17 00:00:00 2001 From: John Feminella Date: Sat, 23 Nov 2013 15:36:09 -0500 Subject: [PATCH 115/162] Fixes broken RST links; clarifies AWS instructions --- docs/sources/installation/amazon.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/sources/installation/amazon.rst b/docs/sources/installation/amazon.rst index 5e92fa8df3..273eceb05f 100644 --- a/docs/sources/installation/amazon.rst +++ b/docs/sources/installation/amazon.rst @@ -22,24 +22,34 @@ Amazon QuickStart 1. **Choose an image:** - * Launch the `Create Instance Wizard` - menu on - your AWS Console + * Launch the `Create Instance Wizard + `_ menu + on your AWS Console. - * Select "Community AMIs" option and serch for ``amd64 precise`` (click enter - to search) + * When picking the source AMI for your instance type, select "Community + AMIs". - * If you choose a EBS enabled AMI you will be able to launch a `t1.micro` - instance (more info on `pricing` ) + * Search for ``amd64 precise``. Pick one of the amd64 Ubuntu images. + + * If you choose a EBS enabled AMI, you'll also be able to launch a + ``t1.micro`` instance (more info on `pricing + `_). ``t1.micro`` instances are + eligible for Amazon's Free Usage Tier. * When you click select you'll be taken to the instance setup, and you're one click away from having your Ubuntu VM up and running. 2. **Tell CloudInit to install Docker:** + * When you're on the "Configure Instance Details" step, expand the "Advanced + Details" section. + + * Under "User data", select "As text". + * Enter ``#include https://get.docker.io`` into the instance *User Data*. `CloudInit `_ is part of the - Ubuntu image you chose and it bootstraps from this *User Data*. + Ubuntu image you chose; it will bootstrap Docker by running the shell + script located at this URL. 3. After a few more standard choices where defaults are probably ok, your AWS Ubuntu instance with Docker should be running! @@ -159,7 +169,7 @@ Docker that way too. Vagrant 1.1 or higher is required. includes rights to SSH (port 22) to your container. If you have an advanced AWS setup, you might want to have a look at - https://github.com/mitchellh/vagrant-aws + `vagrant-aws `_. 7. Connect to your machine From c81bb20f5b2b5d86059c6004e60ba23b03d30fe0 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 2 Dec 2013 16:02:54 -0700 Subject: [PATCH 116/162] Add cgroup-bin dependency to our Ubuntu package Since cgroup-bin is only "recommended" by the lxc package on Ubuntu, but is necessary for having the proper cgroups mounted for Docker to function, this makes some sense for us to add separately. Fixes #2990 --- hack/make/ubuntu | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/make/ubuntu b/hack/make/ubuntu index d4b9fd0b30..ab9644432c 100644 --- a/hack/make/ubuntu +++ b/hack/make/ubuntu @@ -96,6 +96,7 @@ EOF --depends lxc \ --depends aufs-tools \ --depends iptables \ + --depends cgroup-bin \ --description "$PACKAGE_DESCRIPTION" \ --maintainer "$PACKAGE_MAINTAINER" \ --conflicts lxc-docker-virtual-package \ From d9b742419c48bfe57b16d7bf18915a9b31f3548d Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Date: Mon, 2 Dec 2013 18:03:54 -0500 Subject: [PATCH 117/162] Typo fix --- contrib/zfs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zfs/README.md b/contrib/zfs/README.md index 7789180cad..04d36841a3 100644 --- a/contrib/zfs/README.md +++ b/contrib/zfs/README.md @@ -1,4 +1,4 @@ -# ZFS Stroage Driver +# ZFS Storage Driver This is a placeholder to declare the presence and status of ZFS storage driver for containers. From c4c90e9cec1b3f3cc5aa652e4e9155f6a2e687aa Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 29 Nov 2013 17:24:30 -0800 Subject: [PATCH 118/162] Add hostconfig to container inspect --- api.go | 5 ++++- api_params.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index da16044410..70349bd157 100644 --- a/api.go +++ b/api.go @@ -874,7 +874,10 @@ func getContainersByName(srv *Server, version float64, w http.ResponseWriter, r return fmt.Errorf("Conflict between containers and images") } - return writeJSON(w, http.StatusOK, container) + container.readHostConfig() + c := APIContainer{container, container.hostConfig} + + return writeJSON(w, http.StatusOK, c) } func getImagesByName(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/api_params.go b/api_params.go index 8d65d475ce..a1bdd3e147 100644 --- a/api_params.go +++ b/api_params.go @@ -118,6 +118,10 @@ type ( Resource string HostPath string } + APIContainer struct { + *Container + HostConfig *HostConfig + } ) func (api APIImages) ToLegacy() []APIImagesOld { From 62263967b91ca841c43b980aec0ebc74ab2d838d Mon Sep 17 00:00:00 2001 From: Silas Sewell Date: Tue, 3 Dec 2013 06:18:01 +0000 Subject: [PATCH 119/162] Add stream flag to logs command --- commands.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index c86645eee3..f2377d8da8 100644 --- a/commands.go +++ b/commands.go @@ -1506,6 +1506,7 @@ func (cli *DockerCli) CmdDiff(args ...string) error { func (cli *DockerCli) CmdLogs(args ...string) error { cmd := cli.Subcmd("logs", "CONTAINER", "Fetch the logs of a container") + stream := cmd.Bool("stream", false, "Stream output") if err := cmd.Parse(args); err != nil { return nil } @@ -1525,7 +1526,15 @@ func (cli *DockerCli) CmdLogs(args ...string) error { return err } - if err := cli.hijack("POST", "/containers/"+name+"/attach?logs=1&stdout=1&stderr=1", container.Config.Tty, nil, cli.out, cli.err, nil); err != nil { + v := url.Values{} + v.Set("logs", "1") + v.Set("stdout", "1") + v.Set("stderr", "1") + if *stream { + v.Set("stream", "1") + } + + if err := cli.hijack("POST", "/containers/"+name+"/attach?"+v.Encode(), container.Config.Tty, nil, cli.out, cli.err, nil); err != nil { return err } return nil From 3ddbb36a84d5530b1f6b496ff8d79ade6f539267 Mon Sep 17 00:00:00 2001 From: Silas Sewell Date: Tue, 3 Dec 2013 07:17:07 +0000 Subject: [PATCH 120/162] Only stream logs when container is running --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index f2377d8da8..699be1e4cf 100644 --- a/commands.go +++ b/commands.go @@ -1530,7 +1530,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error { v.Set("logs", "1") v.Set("stdout", "1") v.Set("stderr", "1") - if *stream { + if *stream && container.State.Running { v.Set("stream", "1") } From 48e1766527770da5ca4e7554f671705d721a5a04 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 3 Dec 2013 17:57:51 +1000 Subject: [PATCH 121/162] add an example of docker ps, and also of link aliases --- docs/sources/commandline/cli.rst | 9 +++++++++ docs/sources/use/working_with_links_names.rst | 13 +++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index d7bb03b954..3d221f52ef 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -799,6 +799,15 @@ Known Issues (kill) -notrunc=false: Don't truncate output -q=false: Only display numeric IDs +Running ``docker ps`` showing 2 linked containers. + +.. code-block:: bash + + $ docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4c01db0b339c ubuntu:12.04 bash 17 seconds ago Up 16 seconds webapp + d7886598dbe2 crosbymichael/redis:latest /redis-server --dir 33 minutes ago Up 33 minutes 6379/tcp redis,webapp/db + .. _cli_pull: ``pull`` diff --git a/docs/sources/use/working_with_links_names.rst b/docs/sources/use/working_with_links_names.rst index 02906a8226..e6ab9395f8 100644 --- a/docs/sources/use/working_with_links_names.rst +++ b/docs/sources/use/working_with_links_names.rst @@ -54,9 +54,9 @@ inter-container communication is set to false. .. code-block:: bash - # Example: there is an image called redis-2.6 that exposes the port 6379 and starts redis-server. + # Example: there is an image called crosbymichael/redis that exposes the port 6379 and starts redis-server. # Let's name the container as "redis" based on that image and run it as daemon. - $ sudo docker run -d -name redis redis-2.6 + $ sudo docker run -d -name redis crosbymichael/redis We can issue all the commands that you would expect using the name "redis"; start, stop, attach, using the name for our container. The name also allows us to link other containers @@ -102,3 +102,12 @@ about the child container. Accessing the network information along with the environment of the child container allows us to easily connect to the Redis service on the specific IP and port in the environment. + +Running ``docker ps`` shows the 2 containers, and the webapp/db alias name for the redis container. + +.. code-block:: bash + + $ docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4c01db0b339c ubuntu:12.04 bash 17 seconds ago Up 16 seconds webapp + d7886598dbe2 crosbymichael/redis:latest /redis-server --dir 33 minutes ago Up 33 minutes 6379/tcp redis,webapp/db From 682cf48d1db0851789b593db732ed3801e765d56 Mon Sep 17 00:00:00 2001 From: Marek Goldmann Date: Tue, 3 Dec 2013 11:20:29 +0100 Subject: [PATCH 122/162] Instructions on how to install Docker on Red Hat Enterprise Linux / CentOS. --- docs/sources/installation/rhel.rst | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/sources/installation/rhel.rst diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst new file mode 100644 index 0000000000..443d5a4ffc --- /dev/null +++ b/docs/sources/installation/rhel.rst @@ -0,0 +1,65 @@ +:title: Requirements and Installation on Red Hat Enterprise Linux / CentOS +:description: Please note this project is currently under heavy development. It should not be used in production. +:keywords: Docker, Docker documentation, requirements, linux, rhel, centos + +.. _rhel: + +Red Hat Enterprise Linux / CentOS +================================= + +.. include:: install_header.inc + +.. include:: install_unofficial.inc + +Docker is available for **RHEL/CentOS 6**. + +Please note that this package is part of a `Extra Packages for Enterprise Linux (EPEL)`_, a community effort to create and maintain additional packages for RHEL distribution. + +Please note that due to the current Docker limitations Docker is able to run only on the **64 bit** architecture. + +Installation +------------ + +1. Firstly, let's make sure our RHEL host is up-to-date. + +.. code-block:: bash + + sudo yum -y upgrade + +2. Next you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. + +3. Next let's install the ``docker-io`` package which will install Docker on our host. + +.. code-block:: bash + + sudo yum -y install docker-io + +4. Now it's installed lets start the Docker daemon. + +.. code-block:: bash + + sudo service docker start + +If we want Docker to start at boot we should also: + +.. code-block:: bash + + sudo chkconfig docker on + +5. Now let's verify that Docker is working. + +.. code-block:: bash + + sudo docker run -i -t ubuntu /bin/bash + +**Done!**, now continue with the :ref:`hello_world` example. + +Issues? +------- + +If you have any issues - please report them directly in the `Red Hat Bugzilla for docker-io component`_. + +.. _Extra Packages for Enterprise Linux (EPEL): https://fedoraproject.org/wiki/EPEL +.. _EPEL installation instructions: https://fedoraproject.org/wiki/EPEL#How_can_I_use_these_extra_packages.3F +.. _Red Hat Bugzilla for docker-io component : https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora%20EPEL&component=docker-io + From 1cb1e08644b450d3371bfd7d6e305be454d19719 Mon Sep 17 00:00:00 2001 From: Emil Hernvall Date: Thu, 21 Nov 2013 17:26:07 +0100 Subject: [PATCH 123/162] Support for same port on multiple interfaces This commit improves upon the PortMapper and PortAllocator classes by changing their internal data structures for port allocations to use a string rather than a single integer. This string holds the network interface address as well as the port number. This solves a previous problem where a port would be incorrectly reported as being in use because it had been allocated for a different interface. I've also added a basic test case for the PortMapper class, and extended the existing test case for PortAllocator. In the case of PortMapper, this is done by handing it a stub function for creating proxies rather than an actual implementation. --- network.go | 104 +++++++++++++++++++++++++++--------------------- network_test.go | 87 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 139 insertions(+), 52 deletions(-) diff --git a/network.go b/network.go index 1397de0557..fc50a5a907 100644 --- a/network.go +++ b/network.go @@ -225,16 +225,22 @@ func getIfaceAddr(name string) (net.Addr, error) { // up iptables rules. // It keeps track of all mappings and is able to unmap at will type PortMapper struct { - tcpMapping map[int]*net.TCPAddr - tcpProxies map[int]proxy.Proxy - udpMapping map[int]*net.UDPAddr - udpProxies map[int]proxy.Proxy + tcpMapping map[string]*net.TCPAddr + tcpProxies map[string]proxy.Proxy + udpMapping map[string]*net.UDPAddr + udpProxies map[string]proxy.Proxy - iptables *iptables.Chain - defaultIp net.IP + iptables *iptables.Chain + defaultIp net.IP + proxyFactoryFunc func(net.Addr, net.Addr) (proxy.Proxy, error) } func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() + if _, exists := mapper.tcpProxies[mapKey]; exists { + return fmt.Errorf("Port %s is already in use", mapKey) + } + if _, isTCP := backendAddr.(*net.TCPAddr); isTCP { backendPort := backendAddr.(*net.TCPAddr).Port backendIP := backendAddr.(*net.TCPAddr).IP @@ -243,13 +249,13 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { return err } } - mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr) - proxy, err := proxy.NewProxy(&net.TCPAddr{IP: ip, Port: port}, backendAddr) + mapper.tcpMapping[mapKey] = backendAddr.(*net.TCPAddr) + proxy, err := mapper.proxyFactoryFunc(&net.TCPAddr{IP: ip, Port: port}, backendAddr) if err != nil { mapper.Unmap(ip, port, "tcp") return err } - mapper.tcpProxies[port] = proxy + mapper.tcpProxies[mapKey] = proxy go proxy.Run() } else { backendPort := backendAddr.(*net.UDPAddr).Port @@ -259,49 +265,50 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { return err } } - mapper.udpMapping[port] = backendAddr.(*net.UDPAddr) - proxy, err := proxy.NewProxy(&net.UDPAddr{IP: ip, Port: port}, backendAddr) + mapper.udpMapping[mapKey] = backendAddr.(*net.UDPAddr) + proxy, err := mapper.proxyFactoryFunc(&net.UDPAddr{IP: ip, Port: port}, backendAddr) if err != nil { mapper.Unmap(ip, port, "udp") return err } - mapper.udpProxies[port] = proxy + mapper.udpProxies[mapKey] = proxy go proxy.Run() } return nil } func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() if proto == "tcp" { - backendAddr, ok := mapper.tcpMapping[port] + backendAddr, ok := mapper.tcpMapping[mapKey] if !ok { - return fmt.Errorf("Port tcp/%v is not mapped", port) + return fmt.Errorf("Port tcp/%s is not mapped", mapKey) } - if proxy, exists := mapper.tcpProxies[port]; exists { + if proxy, exists := mapper.tcpProxies[mapKey]; exists { proxy.Close() - delete(mapper.tcpProxies, port) + delete(mapper.tcpProxies, mapKey) } if mapper.iptables != nil { if err := mapper.iptables.Forward(iptables.Delete, ip, port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil { return err } } - delete(mapper.tcpMapping, port) + delete(mapper.tcpMapping, mapKey) } else { - backendAddr, ok := mapper.udpMapping[port] + backendAddr, ok := mapper.udpMapping[mapKey] if !ok { - return fmt.Errorf("Port udp/%v is not mapped", port) + return fmt.Errorf("Port udp/%s is not mapped", mapKey) } - if proxy, exists := mapper.udpProxies[port]; exists { + if proxy, exists := mapper.udpProxies[mapKey]; exists { proxy.Close() - delete(mapper.udpProxies, port) + delete(mapper.udpProxies, mapKey) } if mapper.iptables != nil { if err := mapper.iptables.Forward(iptables.Delete, ip, port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil { return err } } - delete(mapper.udpMapping, port) + delete(mapper.udpMapping, mapKey) } return nil } @@ -321,12 +328,13 @@ func newPortMapper(config *DaemonConfig) (*PortMapper, error) { } mapper := &PortMapper{ - tcpMapping: make(map[int]*net.TCPAddr), - tcpProxies: make(map[int]proxy.Proxy), - udpMapping: make(map[int]*net.UDPAddr), - udpProxies: make(map[int]proxy.Proxy), - iptables: chain, - defaultIp: config.DefaultIp, + tcpMapping: make(map[string]*net.TCPAddr), + tcpProxies: make(map[string]proxy.Proxy), + udpMapping: make(map[string]*net.UDPAddr), + udpProxies: make(map[string]proxy.Proxy), + iptables: chain, + defaultIp: config.DefaultIp, + proxyFactoryFunc: proxy.NewProxy, } return mapper, nil } @@ -334,7 +342,7 @@ func newPortMapper(config *DaemonConfig) (*PortMapper, error) { // Port allocator: Automatically allocate and release networking ports type PortAllocator struct { sync.Mutex - inUse map[int]struct{} + inUse map[string]struct{} fountain chan int quit chan bool } @@ -354,20 +362,22 @@ func (alloc *PortAllocator) runFountain() { } // FIXME: Release can no longer fail, change its prototype to reflect that. -func (alloc *PortAllocator) Release(port int) error { +func (alloc *PortAllocator) Release(addr net.IP, port int) error { + mapKey := (&net.TCPAddr{Port: port, IP: addr}).String() utils.Debugf("Releasing %d", port) alloc.Lock() - delete(alloc.inUse, port) + delete(alloc.inUse, mapKey) alloc.Unlock() return nil } -func (alloc *PortAllocator) Acquire(port int) (int, error) { - utils.Debugf("Acquiring %d", port) +func (alloc *PortAllocator) Acquire(addr net.IP, port int) (int, error) { + mapKey := (&net.TCPAddr{Port: port, IP: addr}).String() + utils.Debugf("Acquiring %s", mapKey) if port == 0 { // Allocate a port from the fountain for port := range alloc.fountain { - if _, err := alloc.Acquire(port); err == nil { + if _, err := alloc.Acquire(addr, port); err == nil { return port, nil } } @@ -375,10 +385,10 @@ func (alloc *PortAllocator) Acquire(port int) (int, error) { } alloc.Lock() defer alloc.Unlock() - if _, inUse := alloc.inUse[port]; inUse { + if _, inUse := alloc.inUse[mapKey]; inUse { return -1, fmt.Errorf("Port already in use: %d", port) } - alloc.inUse[port] = struct{}{} + alloc.inUse[mapKey] = struct{}{} return port, nil } @@ -391,7 +401,7 @@ func (alloc *PortAllocator) Close() error { func newPortAllocator() (*PortAllocator, error) { allocator := &PortAllocator{ - inUse: make(map[int]struct{}), + inUse: make(map[string]struct{}), fountain: make(chan int), quit: make(chan bool), } @@ -546,25 +556,25 @@ func (iface *NetworkInterface) AllocatePort(port Port, binding PortBinding) (*Na hostPort, _ := parsePort(nat.Binding.HostPort) if nat.Port.Proto() == "tcp" { - extPort, err := iface.manager.tcpPortAllocator.Acquire(hostPort) + extPort, err := iface.manager.tcpPortAllocator.Acquire(ip, hostPort) if err != nil { return nil, err } backend := &net.TCPAddr{IP: iface.IPNet.IP, Port: containerPort} if err := iface.manager.portMapper.Map(ip, extPort, backend); err != nil { - iface.manager.tcpPortAllocator.Release(extPort) + iface.manager.tcpPortAllocator.Release(ip, extPort) return nil, err } nat.Binding.HostPort = strconv.Itoa(extPort) } else { - extPort, err := iface.manager.udpPortAllocator.Acquire(hostPort) + extPort, err := iface.manager.udpPortAllocator.Acquire(ip, hostPort) if err != nil { return nil, err } backend := &net.UDPAddr{IP: iface.IPNet.IP, Port: containerPort} if err := iface.manager.portMapper.Map(ip, extPort, backend); err != nil { - iface.manager.udpPortAllocator.Release(extPort) + iface.manager.udpPortAllocator.Release(ip, extPort) return nil, err } nat.Binding.HostPort = strconv.Itoa(extPort) @@ -596,16 +606,19 @@ func (iface *NetworkInterface) Release() { continue } ip := net.ParseIP(nat.Binding.HostIp) - utils.Debugf("Unmaping %s/%s", nat.Port.Proto, nat.Binding.HostPort) + utils.Debugf("Unmaping %s/%s:%s", nat.Port.Proto, ip.String(), nat.Binding.HostPort) if err := iface.manager.portMapper.Unmap(ip, hostPort, nat.Port.Proto()); err != nil { log.Printf("Unable to unmap port %s: %s", nat, err) } + if nat.Port.Proto() == "tcp" { - if err := iface.manager.tcpPortAllocator.Release(hostPort); err != nil { + if err := iface.manager.tcpPortAllocator.Release(ip, hostPort); err != nil { log.Printf("Unable to release port %s", nat) } - } else if err := iface.manager.udpPortAllocator.Release(hostPort); err != nil { - log.Printf("Unable to release port %s: %s", nat, err) + } else if nat.Port.Proto() == "udp" { + if err := iface.manager.tcpPortAllocator.Release(ip, hostPort); err != nil { + log.Printf("Unable to release port %s: %s", nat, err) + } } } @@ -732,6 +745,7 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { if err != nil { return nil, err } + udpPortAllocator, err := newPortAllocator() if err != nil { return nil, err diff --git a/network_test.go b/network_test.go index e2631ddcb7..184b497938 100644 --- a/network_test.go +++ b/network_test.go @@ -1,42 +1,52 @@ package docker import ( + "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/proxy" "net" "testing" ) func TestPortAllocation(t *testing.T) { + ip := net.ParseIP("192.168.0.1") + ip2 := net.ParseIP("192.168.0.2") allocator, err := newPortAllocator() if err != nil { t.Fatal(err) } - if port, err := allocator.Acquire(80); err != nil { + if port, err := allocator.Acquire(ip, 80); err != nil { t.Fatal(err) } else if port != 80 { t.Fatalf("Acquire(80) should return 80, not %d", port) } - port, err := allocator.Acquire(0) + port, err := allocator.Acquire(ip, 0) if err != nil { t.Fatal(err) } if port <= 0 { t.Fatalf("Acquire(0) should return a non-zero port") } - if _, err := allocator.Acquire(port); err == nil { + if _, err := allocator.Acquire(ip, port); err == nil { t.Fatalf("Acquiring a port already in use should return an error") } - if newPort, err := allocator.Acquire(0); err != nil { + if newPort, err := allocator.Acquire(ip, 0); err != nil { t.Fatal(err) } else if newPort == port { t.Fatalf("Acquire(0) allocated the same port twice: %d", port) } - if _, err := allocator.Acquire(80); err == nil { + if _, err := allocator.Acquire(ip, 80); err == nil { t.Fatalf("Acquiring a port already in use should return an error") } - if err := allocator.Release(80); err != nil { + if _, err := allocator.Acquire(ip2, 80); err != nil { + t.Fatalf("It should be possible to allocate the same port on a different interface") + } + if _, err := allocator.Acquire(ip2, 80); err == nil { + t.Fatalf("Acquiring a port already in use should return an error") + } + if err := allocator.Release(ip, 80); err != nil { t.Fatal(err) } - if _, err := allocator.Acquire(80); err != nil { + if _, err := allocator.Acquire(ip, 80); err != nil { t.Fatal(err) } } @@ -311,3 +321,66 @@ func TestCheckNameserverOverlaps(t *testing.T) { t.Fatalf("%s should not overlap %v but it does", netX, nameservers) } } + +type StubProxy struct { + frontendAddr *net.Addr + backendAddr *net.Addr +} + +func (proxy *StubProxy) Run() {} +func (proxy *StubProxy) Close() {} +func (proxy *StubProxy) FrontendAddr() net.Addr { return *proxy.frontendAddr } +func (proxy *StubProxy) BackendAddr() net.Addr { return *proxy.backendAddr } + +func NewStubProxy(frontendAddr, backendAddr net.Addr) (proxy.Proxy, error) { + return &StubProxy{ + frontendAddr: &frontendAddr, + backendAddr: &backendAddr, + }, nil +} + +func TestPortMapper(t *testing.T) { + var chain *iptables.Chain + mapper := &PortMapper{ + tcpMapping: make(map[string]*net.TCPAddr), + tcpProxies: make(map[string]proxy.Proxy), + udpMapping: make(map[string]*net.UDPAddr), + udpProxies: make(map[string]proxy.Proxy), + iptables: chain, + defaultIp: net.IP("0.0.0.0"), + proxyFactoryFunc: NewStubProxy, + } + + dstIp1 := net.ParseIP("192.168.0.1") + dstIp2 := net.ParseIP("192.168.0.2") + srcAddr1 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")} + srcAddr2 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.2")} + + if err := mapper.Map(dstIp1, 80, srcAddr1); err != nil { + t.Fatalf("Failed to allocate port: %s", err) + } + + if mapper.Map(dstIp1, 80, srcAddr1) == nil { + t.Fatalf("Port is in use - mapping should have failed") + } + + if mapper.Map(dstIp1, 80, srcAddr2) == nil { + t.Fatalf("Port is in use - mapping should have failed") + } + + if err := mapper.Map(dstIp2, 80, srcAddr2); err != nil { + t.Fatalf("Failed to allocate port: %s", err) + } + + if mapper.Unmap(dstIp1, 80, "tcp") != nil { + t.Fatalf("Failed to release port") + } + + if mapper.Unmap(dstIp2, 80, "tcp") != nil { + t.Fatalf("Failed to release port") + } + + if mapper.Unmap(dstIp2, 80, "tcp") == nil { + t.Fatalf("Port already released, but no error reported") + } +} From bbf9135adcaec9edc2e9d0ebce6a78ba3ade3689 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 22 Oct 2013 20:48:29 +0200 Subject: [PATCH 124/162] Added HTTPAuthDecorator --- utils/http.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/utils/http.go b/utils/http.go index 5eb77d1949..463d3b4fdd 100644 --- a/utils/http.go +++ b/utils/http.go @@ -107,6 +107,23 @@ func (h *HTTPMetaHeadersDecorator) ChangeRequest(req *http.Request) (newReq *htt return req, nil } +type HTTPAuthDecorator struct { + login string + password string +} + +func NewHTTPAuthDecorator(login, password string) HTTPRequestDecorator { + ret := new(HTTPAuthDecorator) + ret.login = login + ret.password = password + return ret +} + +func (self *HTTPAuthDecorator) ChangeRequest(req *http.Request) (*http.Request, error) { + req.SetBasicAuth(self.login, self.password) + return req, nil +} + // HTTPRequestFactory creates an HTTP request // and applies a list of decorators on the request. type HTTPRequestFactory struct { @@ -119,6 +136,10 @@ func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { } } +func (self *HTTPRequestFactory) AddDecorator(d... HTTPRequestDecorator) { + self.decorators = append(self.decorators, d...) +} + // NewRequest() creates a new *http.Request, // applies all decorators in the HTTPRequestFactory on the request, // then applies decorators provided by d on the request. @@ -144,5 +165,6 @@ func (h *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d return nil, err } } + Debugf("%v -- HEADERS: %v", req.URL, req.Header) return req, err } From 045989e3d824f577cd90a6386b66a5814e703766 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 22 Oct 2013 20:49:13 +0200 Subject: [PATCH 125/162] Use basic auth for private registries when over HTTPS. RequestFactory is no longer a singleton (can be different for different instances of Registry) Registry now has an indexEndpoint member Registry methods that needed the indexEndpoint parameter no longer do so Registry methods will only use token auth where applicable if basic auth is not enabled. --- registry/registry.go | 62 +++++++++++++++++++++++++++++---------- registry/registry_test.go | 9 +++--- server.go | 43 ++++++++++++--------------- 3 files changed, 70 insertions(+), 44 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 6aea458e99..c39ecfe5ac 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -160,7 +160,9 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -193,7 +195,9 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return false @@ -209,7 +213,9 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -236,7 +242,9 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -262,7 +270,9 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -290,7 +300,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { +func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { + indexEp := r.indexEndpoint repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) utils.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -364,7 +375,9 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -401,7 +414,9 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { @@ -436,7 +451,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -465,7 +482,9 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -478,8 +497,9 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} + indexEp := r.indexEndpoint if validate { for _, elem := range imgList { @@ -583,6 +603,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { + utils.Debugf("Index server: %s", r.indexEndpoint) u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { @@ -644,12 +665,13 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - reqFactory *utils.HTTPRequestFactory + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory + indexEndpoint string } -func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { +func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -660,12 +682,22 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRe client: &http.Client{ Transport: httpTransport, }, + indexEndpoint: indexEndpoint, } r.client.Jar, err = cookiejar.New(nil) if err != nil { return nil, err } + // If we're working with a private registry over HTTPS, send Basic Auth headers + // alongside our requests. + if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { + utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } + r.reqFactory = factory return r, nil } + diff --git a/registry/registry_test.go b/registry/registry_test.go index fb43da66aa..69eb25b247 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -15,7 +15,7 @@ var ( func spawnTestRegistry(t *testing.T) *Registry { authConfig := &auth.AuthConfig{} - r, err := NewRegistry("", authConfig, utils.NewHTTPRequestFactory()) + r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestGetRemoteTags(t *testing.T) { func TestGetRepositoryData(t *testing.T) { r := spawnTestRegistry(t) - data, err := r.GetRepositoryData(makeURL("/v1/"), "foo42/bar") + data, err := r.GetRepositoryData("foo42/bar") if err != nil { t.Fatal(err) } @@ -168,15 +168,14 @@ func TestPushImageJSONIndex(t *testing.T) { Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } - ep := makeURL("/v1/") - repoData, err := r.PushImageJSONIndex(ep, "foo42/bar", imgData, false, nil) + repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil) if err != nil { t.Fatal(err) } if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex(ep, "foo42/bar", imgData, true, []string{ep}) + repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint}) if err != nil { t.Fatal(err) } diff --git a/server.go b/server.go index a5ab4f2646..40ee7e3c42 100644 --- a/server.go +++ b/server.go @@ -425,7 +425,7 @@ func (srv *Server) recursiveLoad(address, tmpImageDir string) error { } func (srv *Server) ImagesSearch(term string) ([]registry.SearchResult, error) { - r, err := registry.NewRegistry(srv.runtime.config.Root, nil, srv.HTTPRequestFactory(nil)) + r, err := registry.NewRegistry(nil, srv.HTTPRequestFactory(nil), auth.IndexServerAddress()) if err != nil { return nil, err } @@ -816,10 +816,10 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag string, sf *utils.StreamFormatter, parallel bool) error { out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) - repoData, err := r.GetRepositoryData(indexEp, remoteName) + repoData, err := r.GetRepositoryData(remoteName) if err != nil { return err } @@ -989,11 +989,6 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, metaHeaders map[string][]string, parallel bool) error { - r, err := registry.NewRegistry(srv.runtime.config.Root, authConfig, srv.HTTPRequestFactory(metaHeaders)) - if err != nil { - return err - } - out = utils.NewWriteFlusher(out) c, err := srv.poolAdd("pull", localName+":"+tag) @@ -1014,12 +1009,17 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return err } + r, err := registry.NewRegistry(authConfig, srv.HTTPRequestFactory(metaHeaders), endpoint) + if err != nil { + return err + } + if endpoint == auth.IndexServerAddress() { // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" localName = remoteName } - if err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel); err != nil { + if err = srv.pullRepository(r, out, localName, remoteName, tag, sf, parallel); err != nil { return err } @@ -1081,7 +1081,7 @@ func flatten(slc [][]*registry.ImgData) []*registry.ImgData { return result } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) imgList, err := srv.getImageList(localRepo) if err != nil { @@ -1091,7 +1091,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName out.Write(sf.FormatStatus("", "Sending image list")) var repoData *registry.RepositoryData - repoData, err = r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, false, nil) + repoData, err = r.PushImageJSONIndex(remoteName, flattenedImgList, false, nil) if err != nil { return err } @@ -1137,7 +1137,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName } } - if _, err := r.PushImageJSONIndex(indexEp, remoteName, flattenedImgList, true, repoData.Endpoints); err != nil { + if _, err := r.PushImageJSONIndex(remoteName, flattenedImgList, true, repoData.Endpoints); err != nil { return err } @@ -1203,7 +1203,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.config.Root, authConfig, srv.HTTPRequestFactory(metaHeaders)) + r, err2 := registry.NewRegistry(authConfig, srv.HTTPRequestFactory(metaHeaders), endpoint) if err2 != nil { return err2 } @@ -1213,7 +1213,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { - if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { + if err := srv.pushRepository(r, out, localName, remoteName, localRepo, sf); err != nil { return err } return nil @@ -1852,7 +1852,6 @@ func NewServer(eng *engine.Engine, config *DaemonConfig) (*Server, error) { pushingPool: make(map[string]chan struct{}), events: make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events listeners: make(map[string]chan utils.JSONMessage), - reqFactory: nil, } runtime.srv = srv return srv, nil @@ -1861,15 +1860,12 @@ func NewServer(eng *engine.Engine, config *DaemonConfig) (*Server, error) { func (srv *Server) HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { srv.Lock() defer srv.Unlock() - if srv.reqFactory == nil { - ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) - md := &utils.HTTPMetaHeadersDecorator{ - Headers: metaHeaders, - } - factory := utils.NewHTTPRequestFactory(ud, md) - srv.reqFactory = factory + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + md := &utils.HTTPMetaHeadersDecorator{ + Headers: metaHeaders, } - return srv.reqFactory + factory := utils.NewHTTPRequestFactory(ud, md) + return factory } func (srv *Server) LogEvent(action, id, from string) *utils.JSONMessage { @@ -1904,6 +1900,5 @@ type Server struct { pushingPool map[string]chan struct{} events []utils.JSONMessage listeners map[string]chan utils.JSONMessage - reqFactory *utils.HTTPRequestFactory Eng *engine.Engine } From a02bc8a5dbca763a2a245cf0ca17ff5a21d5b52b Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 22 Oct 2013 20:57:48 +0200 Subject: [PATCH 126/162] gofmt --- registry/registry.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index c39ecfe5ac..99f3403a4c 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -665,9 +665,9 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - reqFactory *utils.HTTPRequestFactory + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory indexEndpoint string } @@ -700,4 +700,3 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, r.reqFactory = factory return r, nil } - From ec4863ae5582d8e7a9dccb57d7f5f3e21c2481ef Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 23 Oct 2013 17:56:40 +0200 Subject: [PATCH 127/162] Factorized auth token setting --- registry/registry.go | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 99f3403a4c..ef561fea06 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -153,6 +153,13 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } +func setTokenAuth(req *http.Request, token []string) (*http.Request) { + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } + return req +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -160,9 +167,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -195,9 +200,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return false @@ -213,9 +216,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -242,9 +243,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -375,9 +374,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -414,9 +411,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { @@ -451,9 +446,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -482,9 +475,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { From 3b5010e90bd51ba3630f48091ed70111d26dd31a Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 23 Oct 2013 18:00:40 +0200 Subject: [PATCH 128/162] missed one call to setTokenAuth --- registry/registry.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index ef561fea06..0a8d1ddaa3 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -153,7 +153,7 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -func setTokenAuth(req *http.Request, token []string) (*http.Request) { +func setTokenAuth(req *http.Request, token []string) *http.Request { if req.Header.Get("Authorization") == "" { // Don't override req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) } @@ -269,9 +269,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err From 3f921639894d3eca72c9ae105d3ad4f386651caa Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 4 Nov 2013 21:49:34 +0100 Subject: [PATCH 129/162] Don't return req as result of setTokenAuth --- registry/registry.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 0a8d1ddaa3..6c9255aa46 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -153,11 +153,10 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -func setTokenAuth(req *http.Request, token []string) *http.Request { +func setTokenAuth(req *http.Request, token []string) { if req.Header.Get("Authorization") == "" { // Don't override req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) } - return req } // Retrieve the history of a given image from the Registry. @@ -167,7 +166,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -200,7 +199,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return false @@ -216,7 +215,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -243,7 +242,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -269,7 +268,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -372,7 +371,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -409,7 +408,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { @@ -444,7 +443,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -473,7 +472,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - req = setTokenAuth(req, token) + setTokenAuth(req, token) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { From 9be5db8704be5bda02484c374f4bd9b197d5701b Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 7 Nov 2013 19:36:02 +0100 Subject: [PATCH 130/162] Handle 401 response in auth.Login() for authed private registries --- auth/auth.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/auth/auth.go b/auth/auth.go index 62d8e7f522..85844a5472 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -223,6 +223,28 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e } else { return "", fmt.Errorf("Registration: %s", reqBody) } + } else if reqStatusCode == 401 { + // This case would happen with private registries where /v1/users is + // protected, so people can use `docker login` as an auth check. + req, err := factory.NewRequest("GET", serverAddress+"users/", nil) + req.SetBasicAuth(authConfig.Username, authConfig.Password) + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == 200 { + status = "Login Succeeded" + } else if resp.StatusCode == 401 { + return "", fmt.Errorf("Wrong login/password, please try again") + } else { + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) + } } else { return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) } From b59dea67679fefcfe9716b855ef906b377d9993d Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 3 Dec 2013 10:53:18 -0500 Subject: [PATCH 131/162] Add a proposal step for potential MAINTAINERS. Fixes #3014 --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fed5c61b7..f68270fd81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,6 +122,7 @@ For more details see [MAINTAINERS.md](hack/MAINTAINERS.md) * Step 1: learn the component inside out * Step 2: make yourself useful by contributing code, bugfixes, support etc. * Step 3: volunteer on the irc channel (#docker@freenode) +* Step 4: propose yourself at a scheduled #docker-meeting Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. You don't have to be a maintainer to make a difference on the project! From 5976c26c1e7bc7b29489250b8100412ea3d27b60 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 3 Dec 2013 09:20:14 -0800 Subject: [PATCH 132/162] Ensure that the init layer is removed with the container --- integration/runtime_test.go | 40 +++++++++++++++++++++++++++++++++++++ runtime.go | 5 +++++ 2 files changed, 45 insertions(+) diff --git a/integration/runtime_test.go b/integration/runtime_test.go index 7074a14ce9..016e96d93b 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -843,3 +843,43 @@ func TestGetAllChildren(t *testing.T) { } } } + +func TestDestroyWithInitLayer(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + container, _, err := runtime.Create(&docker.Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"ls", "-al"}, + }, "") + + if err != nil { + t.Fatal(err) + } + // Destroy + if err := runtime.Destroy(container); err != nil { + t.Fatal(err) + } + + // Make sure runtime.Exists() behaves correctly + if runtime.Exists("test_destroy") { + t.Fatalf("Exists() returned true") + } + + // Make sure runtime.List() doesn't list the destroyed container + if len(runtime.List()) != 0 { + t.Fatalf("Expected 0 container, %v found", len(runtime.List())) + } + + driver := runtime.Graph().Driver() + + // Make sure that the container does not exist in the driver + if _, err := driver.Get(container.ID); err == nil { + t.Fatal("Conttainer should not exist in the driver") + } + + // Make sure that the init layer is removed from the driver + if _, err := driver.Get(fmt.Sprintf("%s-init", container.ID)); err == nil { + t.Fatal("Container's init layer should not exist in the driver") + } +} diff --git a/runtime.go b/runtime.go index a7c2659b00..2e2492a6e1 100644 --- a/runtime.go +++ b/runtime.go @@ -237,6 +237,11 @@ func (runtime *Runtime) Destroy(container *Container) error { return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", runtime.driver, container.ID, err) } + initID := fmt.Sprintf("%s-init", container.ID) + if err := runtime.driver.Remove(initID); err != nil { + return fmt.Errorf("Driver %s failed to remove init filesystem %s: %s", runtime.driver, initID, err) + } + if _, err := runtime.containerGraph.Purge(container.ID); err != nil { utils.Debugf("Unable to remove container from link graph: %s", err) } From 6a55169e2e0d039f7404b236b321fa7c76b99618 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 3 Dec 2013 13:04:18 -0500 Subject: [PATCH 133/162] Expose expvar endpoint during debugging. Fixes #3017 --- api.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/api.go b/api.go index da16044410..62486ca9d8 100644 --- a/api.go +++ b/api.go @@ -6,6 +6,7 @@ import ( "code.google.com/p/go.net/websocket" "encoding/base64" "encoding/json" + "expvar" "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" @@ -1063,7 +1064,23 @@ func makeHttpHandler(srv *Server, logging bool, localMethod string, localRoute s } } +// Replicated from expvar.go as not public. +func expvarHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "{\n") + first := true + expvar.Do(func(kv expvar.KeyValue) { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) + }) + fmt.Fprintf(w, "\n}\n") +} + func AttachProfiler(router *mux.Router) { + router.HandleFunc("/debug/vars", expvarHandler) router.HandleFunc("/debug/pprof/", pprof.Index) router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) router.HandleFunc("/debug/pprof/profile", pprof.Profile) From d21563ced3a24ebfc8a9b53e5ec687543f7b2979 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 3 Dec 2013 13:53:57 -0500 Subject: [PATCH 134/162] Support TESTFLAGS --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dd992946a4..79513f0126 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ doc: cd docs && docker build -t docker-docs . && docker run -p 8000:8000 docker-docs test: bundles - docker run -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh test + docker run -e TESTFLAGS -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh test shell: docker run -privileged -i -t docker bash From 664174c7aa1300b9a856eafafc01a1d416a731ff Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 2 Dec 2013 18:17:15 -0800 Subject: [PATCH 135/162] Add docs for hostconfig in inspect --- docs/sources/api/docker_remote_api.rst | 4 ++++ docs/sources/api/docker_remote_api_v1.8.rst | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 4191e9a8e1..f38a36ee1f 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -51,6 +51,10 @@ What's new **New!** This endpoint now returns build status as json stream. In case of a build error, it returns the exit status of the failed command. +.. http:get:: /containers/(id)/json + + **New!** This endpoint now returns the host config for the container. + v1.7 **** diff --git a/docs/sources/api/docker_remote_api_v1.8.rst b/docs/sources/api/docker_remote_api_v1.8.rst index 8e960739ba..1e02817611 100644 --- a/docs/sources/api/docker_remote_api_v1.8.rst +++ b/docs/sources/api/docker_remote_api_v1.8.rst @@ -222,7 +222,23 @@ Inspect a container }, "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", "ResolvConfPath": "/etc/resolv.conf", - "Volumes": {} + "Volumes": {}, + "HostConfig": { + "Binds": null, + "ContainerIDFile": "", + "LxcConf": [], + "Privileged": false, + "PortBindings": { + "80/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "49153" + } + ] + }, + "Links": null, + "PublishAllPorts": false + } } :statuscode 200: no error From 64439505c7801214d4e01941c2e92d64261ab137 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 3 Dec 2013 14:17:59 -0500 Subject: [PATCH 136/162] Added note about Ubuntu curl installation --- docs/sources/installation/ubuntulinux.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 29e009d90c..93a1b28e2a 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -86,6 +86,14 @@ continue installation.* sudo apt-get update sudo apt-get install lxc-docker +.. note:: + + There is also a simple ``curl`` script available to help with this process. + + .. code-block:: bash + + curl -s http://get.docker.io/ubuntu/ | sudo sh + Now verify that the installation has worked by downloading the ``ubuntu`` image and launching a container. From b699aee91f90b8390dfa6e8ef58e983fc8c4f3e5 Mon Sep 17 00:00:00 2001 From: Silas Sewell Date: Tue, 3 Dec 2013 20:35:22 +0000 Subject: [PATCH 137/162] Rename logs -stream to logs -f --- commands.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 699be1e4cf..c024333a65 100644 --- a/commands.go +++ b/commands.go @@ -1506,7 +1506,7 @@ func (cli *DockerCli) CmdDiff(args ...string) error { func (cli *DockerCli) CmdLogs(args ...string) error { cmd := cli.Subcmd("logs", "CONTAINER", "Fetch the logs of a container") - stream := cmd.Bool("stream", false, "Stream output") + follow := cmd.Bool("f", false, "Follow log output") if err := cmd.Parse(args); err != nil { return nil } @@ -1530,7 +1530,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error { v.Set("logs", "1") v.Set("stdout", "1") v.Set("stderr", "1") - if *stream && container.State.Running { + if *follow && container.State.Running { v.Set("stream", "1") } From 45567f2209d3450142ae8042f674269902d58e3b Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Mon, 25 Nov 2013 22:14:33 -0500 Subject: [PATCH 138/162] Updated Fedora docs * Added Fedora installation instructions --- docs/sources/installation/fedora.rst | 41 +++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index 46908f0139..88b1037bfb 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -11,9 +11,42 @@ Fedora .. include:: install_unofficial.inc -.. warning:: +Docker is available in **Fedora 19 and later**. Please note that due to the +current Docker limitations Docker is able to run only on the **64 bit** +architecture. - This is a placeholder for the Fedora installation instructions. Currently there is not an available - Docker package in the Fedora distribution. These packages are being built and should be available soon. - These instructions will be updated when the package is available. +Installation +------------ + +Firstly, let's make sure our Fedora host is up-to-date. + +.. code-block:: bash + + sudo yum -y upgrade + +Next let's install the ``docker-io`` package which will install Docker on our host. + +.. code-block:: bash + + sudo yum -y install docker-io + +Now it's installed lets start the Docker daemon. + +.. code-block:: bash + + sudo systemctl start docker + +If we want Docker to start at boot we should also: + +.. code-block:: bash + + sudo systemctl enable docker + +Now let's verify that Docker is working. + +.. code-block:: bash + + sudo docker run -i -t ubuntu /bin/bash + +**Done!**, now continue with the :ref:`hello_world` example. From f50fe14e13160c7c823613710a6fa12c9fc13bcb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 2 Dec 2013 11:53:04 -0800 Subject: [PATCH 139/162] Handle small screens --- commands.go | 2 +- utils/jsonmessage.go | 48 +++++++++++++++++++++++++++++++-------- utils/jsonmessage_test.go | 13 +++++++---- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/commands.go b/commands.go index c86645eee3..e81faf12ec 100644 --- a/commands.go +++ b/commands.go @@ -2294,7 +2294,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h } if matchesContentType(resp.Header.Get("Content-Type"), "application/json") { - return utils.DisplayJSONMessagesStream(resp.Body, out, cli.isTerminal) + return utils.DisplayJSONMessagesStream(resp.Body, out, cli.terminalFd, cli.isTerminal) } if _, err := io.Copy(out, resp.Body); err != nil { return err diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index 809524cd28..eaa90a5b64 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -3,6 +3,7 @@ package utils import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/term" "io" "strings" "time" @@ -18,27 +19,50 @@ func (e *JSONError) Error() string { } type JSONProgress struct { - Current int `json:"current,omitempty"` - Total int `json:"total,omitempty"` - Start int64 `json:"start,omitempty"` + terminalFd uintptr + Current int `json:"current,omitempty"` + Total int `json:"total,omitempty"` + Start int64 `json:"start,omitempty"` } func (p *JSONProgress) String() string { + var ( + width = 200 + pbBox string + numbersBox string + timeLeftBox string + ) + + ws, err := term.GetWinsize(p.terminalFd) + if err == nil { + width = int(ws.Width) + } + if p.Current == 0 && p.Total == 0 { return "" } current := HumanSize(int64(p.Current)) if p.Total == 0 { - return fmt.Sprintf("%8v/?", current) + return fmt.Sprintf("%8v", current) } total := HumanSize(int64(p.Total)) percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if width > 110 { + pbBox = fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", 50-percentage)) + } + numbersBox = fmt.Sprintf("%8v/%v", current, total) - fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) - perEntry := fromStart / time.Duration(p.Current) - left := time.Duration(p.Total-p.Current) * perEntry - left = (left / time.Second) * time.Second - return fmt.Sprintf("[%s>%s] %8v/%v %s", strings.Repeat("=", percentage), strings.Repeat(" ", 50-percentage), current, total, left.String()) + if p.Start > 0 { + fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) + perEntry := fromStart / time.Duration(p.Current) + left := time.Duration(p.Total-p.Current) * perEntry + left = (left / time.Second) * time.Second + + if width > 50 { + timeLeftBox = " " + left.String() + } + } + return pbBox + numbersBox + timeLeftBox } type JSONMessage struct { @@ -84,7 +108,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { return nil } -func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) error { +func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr, isTerminal bool) error { var ( dec = json.NewDecoder(in) ids = make(map[string]int) @@ -98,6 +122,10 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) err } return err } + + if jm.Progress != nil { + jm.Progress.terminalFd = terminalFd + } if (jm.Progress != nil || jm.ProgressMessage != "") && jm.ID != "" { line, ok := ids[jm.ID] if !ok { diff --git a/utils/jsonmessage_test.go b/utils/jsonmessage_test.go index 9421d97a86..ecf1896762 100644 --- a/utils/jsonmessage_test.go +++ b/utils/jsonmessage_test.go @@ -12,13 +12,18 @@ func TestError(t *testing.T) { } func TestProgress(t *testing.T) { - jp := JSONProgress{0, 0, 0} + jp := JSONProgress{} if jp.String() != "" { t.Fatalf("Expected empty string, got '%s'", jp.String()) } - jp2 := JSONProgress{1, 0, 0} - if jp2.String() != " 1 B/?" { - t.Fatalf("Expected ' 1/?', got '%s'", jp2.String()) + jp2 := JSONProgress{Current: 1} + if jp2.String() != " 1 B" { + t.Fatalf("Expected ' 1 B', got '%s'", jp2.String()) + } + + jp3 := JSONProgress{Current: 50, Total: 100} + if jp3.String() != "[=========================> ] 50 B/100 B" { + t.Fatalf("Expected '[=========================> ] 50 B/100 B', got '%s'", jp3.String()) } } From 84f78d9cad4162ddfc1bccbe55402050123cb1c5 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 3 Dec 2013 17:35:54 -0500 Subject: [PATCH 140/162] Extract helper method for volume linking. Makes this more readable. --- container.go | 76 +++++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/container.go b/container.go index 662043db8a..37c5086650 100644 --- a/container.go +++ b/container.go @@ -583,42 +583,7 @@ func (container *Container) Start() (err error) { } // Apply volumes from another container if requested - if container.Config.VolumesFrom != "" { - containerSpecs := strings.Split(container.Config.VolumesFrom, ",") - for _, containerSpec := range containerSpecs { - mountRW := true - specParts := strings.SplitN(containerSpec, ":", 2) - switch len(specParts) { - case 0: - return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom) - case 2: - switch specParts[1] { - case "ro": - mountRW = false - case "rw": // mountRW is already true - default: - return fmt.Errorf("Malformed volumes-from speficication: %s", containerSpec) - } - } - c := container.runtime.Get(specParts[0]) - if c == nil { - return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID) - } - for volPath, id := range c.Volumes { - if _, exists := container.Volumes[volPath]; exists { - continue - } - if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil { - return err - } - container.Volumes[volPath] = id - if isRW, exists := c.VolumesRW[volPath]; exists { - container.VolumesRW[volPath] = isRW && mountRW - } - } - - } - } + container.joinVolumes() volumesDriver := container.runtime.volumes.driver // Create the requested volumes if they don't exist @@ -915,6 +880,45 @@ func (container *Container) Start() (err error) { return ErrContainerStart } +func (container *Container) joinVolumes() error { + if container.Config.VolumesFrom != "" { + containerSpecs := strings.Split(container.Config.VolumesFrom, ",") + for _, containerSpec := range containerSpecs { + mountRW := true + specParts := strings.SplitN(containerSpec, ":", 2) + switch len(specParts) { + case 0: + return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom) + case 2: + switch specParts[1] { + case "ro": + mountRW = false + case "rw": // mountRW is already true + default: + return fmt.Errorf("Malformed volumes-from speficication: %s", containerSpec) + } + } + c := container.runtime.Get(specParts[0]) + if c == nil { + return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID) + } + for volPath, id := range c.Volumes { + if _, exists := container.Volumes[volPath]; exists { + continue + } + if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil { + return err + } + container.Volumes[volPath] = id + if isRW, exists := c.VolumesRW[volPath]; exists { + container.VolumesRW[volPath] = isRW && mountRW + } + } + + } + } +} + func (container *Container) Run() error { if err := container.Start(); err != nil { return err From 0132547a385697f3fee1e6bf9fe3fa07c98f3fd0 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 3 Dec 2013 17:51:34 -0500 Subject: [PATCH 141/162] Added myself to docs MAINTAINERS --- docs/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index a506ce11df..8ef58199e7 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,2 +1,3 @@ Andy Rothfusz (@metalivedev) Ken Cochrane (@kencochrane) +James Turnbull (@jamesturnbull) From 40fe9f581bd56bbcab9f231763cfac20e659d8c3 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Tue, 3 Dec 2013 17:47:49 -0500 Subject: [PATCH 142/162] Extract volume bind, creation and external methods. Make Start() slightly more readable. --- container.go | 313 +++++++++++++++++++++++++++------------------------ 1 file changed, 165 insertions(+), 148 deletions(-) diff --git a/container.go b/container.go index 37c5086650..bb0723eada 100644 --- a/container.go +++ b/container.go @@ -541,160 +541,18 @@ func (container *Container) Start() (err error) { log.Printf("WARNING: IPv4 forwarding is disabled. Networking will not work") } - // Create the requested bind mounts - binds := make(map[string]BindMap) - // Define illegal container destinations - illegalDsts := []string{"/", "."} - - for _, bind := range container.hostConfig.Binds { - // FIXME: factorize bind parsing in parseBind - var src, dst, mode string - arr := strings.Split(bind, ":") - if len(arr) == 2 { - src = arr[0] - dst = arr[1] - mode = "rw" - } else if len(arr) == 3 { - src = arr[0] - dst = arr[1] - mode = arr[2] - } else { - return fmt.Errorf("Invalid bind specification: %s", bind) - } - - // Bail if trying to mount to an illegal destination - for _, illegal := range illegalDsts { - if dst == illegal { - return fmt.Errorf("Illegal bind destination: %s", dst) - } - } - - bindMap := BindMap{ - SrcPath: src, - DstPath: dst, - Mode: mode, - } - binds[path.Clean(dst)] = bindMap - } - if container.Volumes == nil || len(container.Volumes) == 0 { container.Volumes = make(map[string]string) container.VolumesRW = make(map[string]bool) } // Apply volumes from another container if requested - container.joinVolumes() + if err := container.applyExternalVolumes(); err != nil { + return err + } - volumesDriver := container.runtime.volumes.driver - // Create the requested volumes if they don't exist - for volPath := range container.Config.Volumes { - volPath = path.Clean(volPath) - volIsDir := true - // Skip existing volumes - if _, exists := container.Volumes[volPath]; exists { - continue - } - var srcPath string - var isBindMount bool - srcRW := false - // If an external bind is defined for this volume, use that as a source - if bindMap, exists := binds[volPath]; exists { - isBindMount = true - srcPath = bindMap.SrcPath - if strings.ToLower(bindMap.Mode) == "rw" { - srcRW = true - } - if file, err := os.Open(bindMap.SrcPath); err != nil { - return err - } else { - defer file.Close() - if stat, err := file.Stat(); err != nil { - return err - } else { - volIsDir = stat.IsDir() - } - } - // Otherwise create an directory in $ROOT/volumes/ and use that - } else { - - // Do not pass a container as the parameter for the volume creation. - // The graph driver using the container's information ( Image ) to - // create the parent. - c, err := container.runtime.volumes.Create(nil, nil, "", "", nil) - if err != nil { - return err - } - srcPath, err = volumesDriver.Get(c.ID) - if err != nil { - return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err) - } - srcRW = true // RW by default - } - container.Volumes[volPath] = srcPath - container.VolumesRW[volPath] = srcRW - // Create the mountpoint - rootVolPath := path.Join(container.RootfsPath(), volPath) - if volIsDir { - if err := os.MkdirAll(rootVolPath, 0755); err != nil { - return err - } - } - - volPath = path.Join(container.RootfsPath(), volPath) - if _, err := os.Stat(volPath); err != nil { - if os.IsNotExist(err) { - if volIsDir { - if err := os.MkdirAll(volPath, 0755); err != nil { - return err - } - } else { - if err := os.MkdirAll(path.Dir(volPath), 0755); err != nil { - return err - } - if f, err := os.OpenFile(volPath, os.O_CREATE, 0755); err != nil { - return err - } else { - f.Close() - } - } - } - } - - // Do not copy or change permissions if we are mounting from the host - if srcRW && !isBindMount { - volList, err := ioutil.ReadDir(rootVolPath) - if err != nil { - return err - } - if len(volList) > 0 { - srcList, err := ioutil.ReadDir(srcPath) - if err != nil { - return err - } - if len(srcList) == 0 { - // If the source volume is empty copy files from the root into the volume - if err := archive.CopyWithTar(rootVolPath, srcPath); err != nil { - return err - } - - var stat syscall.Stat_t - if err := syscall.Stat(rootVolPath, &stat); err != nil { - return err - } - var srcStat syscall.Stat_t - if err := syscall.Stat(srcPath, &srcStat); err != nil { - return err - } - // Change the source volume's ownership if it differs from the root - // files that where just copied - if stat.Uid != srcStat.Uid || stat.Gid != srcStat.Gid { - if err := os.Chown(srcPath, int(stat.Uid), int(stat.Gid)); err != nil { - return err - } - } - } - } - } + if err := container.createVolumes(); err != nil { + return err } if err := container.generateLXCConfig(); err != nil { @@ -880,7 +738,165 @@ func (container *Container) Start() (err error) { return ErrContainerStart } -func (container *Container) joinVolumes() error { +func (container *Container) getBindMap() (map[string]BindMap, error) { + // Create the requested bind mounts + binds := make(map[string]BindMap) + // Define illegal container destinations + illegalDsts := []string{"/", "."} + + for _, bind := range container.hostConfig.Binds { + // FIXME: factorize bind parsing in parseBind + var src, dst, mode string + arr := strings.Split(bind, ":") + if len(arr) == 2 { + src = arr[0] + dst = arr[1] + mode = "rw" + } else if len(arr) == 3 { + src = arr[0] + dst = arr[1] + mode = arr[2] + } else { + return nil, fmt.Errorf("Invalid bind specification: %s", bind) + } + + // Bail if trying to mount to an illegal destination + for _, illegal := range illegalDsts { + if dst == illegal { + return nil, fmt.Errorf("Illegal bind destination: %s", dst) + } + } + + bindMap := BindMap{ + SrcPath: src, + DstPath: dst, + Mode: mode, + } + binds[path.Clean(dst)] = bindMap + } + return binds, nil +} + +func (container *Container) createVolumes() error { + binds, err := container.getBindMap() + if err != nil { + return err + } + volumesDriver := container.runtime.volumes.driver + // Create the requested volumes if they don't exist + for volPath := range container.Config.Volumes { + volPath = path.Clean(volPath) + volIsDir := true + // Skip existing volumes + if _, exists := container.Volumes[volPath]; exists { + continue + } + var srcPath string + var isBindMount bool + srcRW := false + // If an external bind is defined for this volume, use that as a source + if bindMap, exists := binds[volPath]; exists { + isBindMount = true + srcPath = bindMap.SrcPath + if strings.ToLower(bindMap.Mode) == "rw" { + srcRW = true + } + if file, err := os.Open(bindMap.SrcPath); err != nil { + return err + } else { + defer file.Close() + if stat, err := file.Stat(); err != nil { + return err + } else { + volIsDir = stat.IsDir() + } + } + // Otherwise create an directory in $ROOT/volumes/ and use that + } else { + + // Do not pass a container as the parameter for the volume creation. + // The graph driver using the container's information ( Image ) to + // create the parent. + c, err := container.runtime.volumes.Create(nil, nil, "", "", nil) + if err != nil { + return err + } + srcPath, err = volumesDriver.Get(c.ID) + if err != nil { + return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err) + } + srcRW = true // RW by default + } + container.Volumes[volPath] = srcPath + container.VolumesRW[volPath] = srcRW + // Create the mountpoint + rootVolPath := path.Join(container.RootfsPath(), volPath) + if volIsDir { + if err := os.MkdirAll(rootVolPath, 0755); err != nil { + return err + } + } + + volPath = path.Join(container.RootfsPath(), volPath) + if _, err := os.Stat(volPath); err != nil { + if os.IsNotExist(err) { + if volIsDir { + if err := os.MkdirAll(volPath, 0755); err != nil { + return err + } + } else { + if err := os.MkdirAll(path.Dir(volPath), 0755); err != nil { + return err + } + if f, err := os.OpenFile(volPath, os.O_CREATE, 0755); err != nil { + return err + } else { + f.Close() + } + } + } + } + + // Do not copy or change permissions if we are mounting from the host + if srcRW && !isBindMount { + volList, err := ioutil.ReadDir(rootVolPath) + if err != nil { + return err + } + if len(volList) > 0 { + srcList, err := ioutil.ReadDir(srcPath) + if err != nil { + return err + } + if len(srcList) == 0 { + // If the source volume is empty copy files from the root into the volume + if err := archive.CopyWithTar(rootVolPath, srcPath); err != nil { + return err + } + + var stat syscall.Stat_t + if err := syscall.Stat(rootVolPath, &stat); err != nil { + return err + } + var srcStat syscall.Stat_t + if err := syscall.Stat(srcPath, &srcStat); err != nil { + return err + } + // Change the source volume's ownership if it differs from the root + // files that where just copied + if stat.Uid != srcStat.Uid || stat.Gid != srcStat.Gid { + if err := os.Chown(srcPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + } + } + } + } + } + return nil +} + +func (container *Container) applyExternalVolumes() error { if container.Config.VolumesFrom != "" { containerSpecs := strings.Split(container.Config.VolumesFrom, ",") for _, containerSpec := range containerSpecs { @@ -917,6 +933,7 @@ func (container *Container) joinVolumes() error { } } + return nil } func (container *Container) Run() error { From 58c33360b01748f4900dd00dc57c3aec89a98e27 Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Tue, 3 Dec 2013 16:39:23 -0800 Subject: [PATCH 143/162] - Updated the doc with the current implementation status --- docs/sources/api/registry_index_spec.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/sources/api/registry_index_spec.rst b/docs/sources/api/registry_index_spec.rst index 78b61c816a..58b205672c 100644 --- a/docs/sources/api/registry_index_spec.rst +++ b/docs/sources/api/registry_index_spec.rst @@ -37,7 +37,6 @@ We expect that there will be only one instance of the index, run and managed by - It delegates authentication and authorization to the Index Auth service using tokens - It supports different storage backends (S3, cloud files, local FS) - It doesn’t have a local database -- It will be open-sourced at some point We expect that there will be multiple registries out there. To help to grasp the context, here are some examples of registries: @@ -46,10 +45,6 @@ We expect that there will be multiple registries out there. To help to grasp the - **vendor registry**: such a registry is provided by a software vendor, who wants to distribute docker images. It would be operated and managed by the vendor. Only users authorized by the vendor would be able to get write access. Some images would be public (accessible for anyone), others private (accessible only for authorized users). Authentication and authorization would be delegated to the Index. The goal of vendor registries is to let someone do “docker pull basho/riak1.3” and automatically push from the vendor registry (instead of a sponsor registry); i.e. get all the convenience of a sponsor registry, while retaining control on the asset distribution. - **private registry**: such a registry is located behind a firewall, or protected by an additional security layer (HTTP authorization, SSL client-side certificates, IP address authorization...). The registry is operated by a private entity, outside of dotCloud’s control. It can optionally delegate additional authorization to the Index, but it is not mandatory. -.. note:: - - Mirror registries and private registries which do not use the Index don’t even need to run the registry code. They can be implemented by any kind of transport implementing HTTP GET and PUT. Read-only registries can be powered by a simple static HTTP server. - .. note:: The latter implies that while HTTP is the protocol of choice for a registry, multiple schemes are possible (and in some cases, trivial): From 6ddea783ef8067b0c739785b6afd10d58255b4c9 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 4 Dec 2013 16:47:45 +1000 Subject: [PATCH 144/162] initial stab at writing a use/ambassador-pattern howto (Issue #2485) --- .../use/ambassador_pattern_linking.rst | 182 ++++++++++++++++++ docs/sources/use/index.rst | 1 + 2 files changed, 183 insertions(+) create mode 100644 docs/sources/use/ambassador_pattern_linking.rst diff --git a/docs/sources/use/ambassador_pattern_linking.rst b/docs/sources/use/ambassador_pattern_linking.rst new file mode 100644 index 0000000000..98eb009d81 --- /dev/null +++ b/docs/sources/use/ambassador_pattern_linking.rst @@ -0,0 +1,182 @@ +:title: Ambassador pattern linking +:description: Using the Ambassador pattern to abstract (network) services +:keywords: Examples, Usage, links, docker, documentation, examples, names, name, container naming + +.. _ambassador_pattern_linking: + +Ambassador pattern linking +========================== + +Rather than hardcoding network links between a service consumer and provider, Docker +encourages service portability. + +eg, instead of + +.. code-block:: bash + + (consumer) --> (redis) + +requiring you to restart the ``consumer`` to attach it to a different ``redis`` service, +you can add ambassadors + +.. code-block:: bash + + (consumer) --> (redis-ambassador) --> (redis) + + or + + (consumer) --> (redis-ambassador) ---network---> (redis-ambassador) --> (redis) + +When you need to rewire your consumer to talk to a different resdis server, you +can just restart the ``redis-ambassador`` container that the consumer is connected to. + +This pattern also allows you to transparently move the redis server to a different +docker host from the consumer. + +Using the ``svendowideit/ambassador`` container, the link wiring is controlled entirely +from the ``docker run`` parameters. + +Two host Example +---------------- + +Start actual redis server on one Docker host + +.. code-block:: bash + + big-server $ docker run -d -name redis crosbymichael/redis + +Then add an ambassador linked to the redis server, mapping a port to the outside world + +.. code-block:: bash + + big-server $ docker run -d -link redis:redis -name redis_ambassador -p 6379:6379 svendowideit/ambassador + +On the other host, you can set up another ambassador setting environment variables for each remote port we want to proxy to the ``big-server`` + +.. code-block:: bash + + client-server $ docker run -d -name redis_ambassador -expose 6379 -e REDIS_PORT_6379_TCP=tcp://192.168.1.52:6379 svendowideit/ambassador + +Then on the ``client-server`` host, you can use a redis client container to talk +to the remote redis server, just by linking to the local redis ambassador. + +.. code-block:: bash + + client-server $ docker run -i -t -rm -link redis_ambassador:redis relateiq/redis-cli + redis 172.17.0.160:6379> ping + PONG + + + +How it works +------------ + +The following example shows what the ``svendowideit/ambassador`` container does +automatically (with a tiny amount of ``sed``) + +On the docker host (192.168.1.52) that redis will run on: + +.. code-block:: bash + + # start actual redis server + $ docker run -d -name redis crosbymichael/redis + + # get a redis-cli container for connection testing + $ docker pull relateiq/redis-cli + + # test the redis server by talking to it directly + $ docker run -t -i -rm -link redis:redis relateiq/redis-cli + redis 172.17.0.136:6379> ping + PONG + ^D + + # add redis ambassador + $ docker run -t -i -link redis:redis -name redis_ambassador -p 6379:6379 busybox sh + +in the redis_ambassador container, you can see the linked redis containers's env + +.. code-block:: bash + + $ env + REDIS_PORT=tcp://172.17.0.136:6379 + REDIS_PORT_6379_TCP_ADDR=172.17.0.136 + REDIS_NAME=/redis_ambassador/redis + HOSTNAME=19d7adf4705e + REDIS_PORT_6379_TCP_PORT=6379 + HOME=/ + REDIS_PORT_6379_TCP_PROTO=tcp + container=lxc + REDIS_PORT_6379_TCP=tcp://172.17.0.136:6379 + TERM=xterm + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PWD=/ + + +This environment is used by the ambassador socat script to expose redis to the world +(via the -p 6379:6379 port mapping) + +.. code-block:: bash + + $ docker rm redis_ambassador + $ sudo ./contrib/mkimage-unittest.sh + $ docker run -t -i -link redis:redis -name redis_ambassador -p 6379:6379 docker-ut sh + + $ socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:172.17.0.136:6379 + +then ping the redis server via the ambassador + +.. code-block::bash + + $ docker run -i -t -rm -link redis_ambassador:redis relateiq/redis-cli + redis 172.17.0.160:6379> ping + PONG + +Now goto a different server + +.. code-block:: bash + + $ sudo ./contrib/mkimage-unittest.sh + $ docker run -t -i -expose 6379 -name redis_ambassador docker-ut sh + + $ socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:192.168.1.52:6379 + +and get the redis-cli image so we can talk over the ambassador bridge + +.. code-block:: bash + + $ docker pull relateiq/redis-cli + $ docker run -i -t -rm -link redis_ambassador:redis relateiq/redis-cli + redis 172.17.0.160:6379> ping + PONG + +The svendowideit/ambassador Dockerfile +-------------------------------------- + +The ``svendowideit/ambassador`` image is a small busybox image with ``socat`` built in. +When you start the container, it uses a small ``sed`` script to parse out the (possibly multiple) +link environment variables to set up the port forwarding. On the remote host, you need to set the +variable using the ``-e`` command line option. + +``-expose 1234 -e REDIS_PORT_1234_TCP=tcp://192.168.1.52:6379`` will forward the +local ``1234`` port to the remote IP and port - in this case ``192.168.1.52:6379``. + + +.. code-block:: Dockerfile + + # + # + # first you need to build the docker-ut image using ./contrib/mkimage-unittest.sh + # then + # docker build -t SvenDowideit/ambassador . + # docker tag SvenDowideit/ambassador ambassador + # then to run it (on the host that has the real backend on it) + # docker run -t -i -link redis:redis -name redis_ambassador -p 6379:6379 ambassador + # on the remote host, you can set up another ambassador + # docker run -t -i -name redis_ambassador -expose 6379 sh + + FROM docker-ut + MAINTAINER SvenDowideit@home.org.au + + + CMD env | grep _TCP= | sed 's/.*_PORT_\([0-9]*\)_TCP=tcp:\/\/\(.*\):\(.*\)/socat TCP4-LISTEN:\1,fork,reuseaddr TCP4:\2:\3 \&/' | sh && top + diff --git a/docs/sources/use/index.rst b/docs/sources/use/index.rst index 38f61ba744..a3e595a1b8 100644 --- a/docs/sources/use/index.rst +++ b/docs/sources/use/index.rst @@ -21,3 +21,4 @@ Contents: host_integration working_with_volumes working_with_links_names + ambassador_pattern_linking From 12180948be8040a4cdf99a0e660098cd33e32832 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 4 Dec 2013 11:54:11 -0800 Subject: [PATCH 145/162] remove unused parameter in Download --- api.go | 2 +- buildfile.go | 2 +- server.go | 4 ++-- utils/utils.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 62486ca9d8..ab9990aa45 100644 --- a/api.go +++ b/api.go @@ -932,7 +932,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ } context = c } else if utils.IsURL(remoteURL) { - f, err := utils.Download(remoteURL, ioutil.Discard) + f, err := utils.Download(remoteURL) if err != nil { return err } diff --git a/buildfile.go b/buildfile.go index e8f178fbbd..170d739ee4 100644 --- a/buildfile.go +++ b/buildfile.go @@ -258,7 +258,7 @@ func (b *buildFile) CmdVolume(args string) error { } func (b *buildFile) addRemote(container *Container, orig, dest string) error { - file, err := utils.Download(orig, ioutil.Discard) + file, err := utils.Download(orig) if err != nil { return err } diff --git a/server.go b/server.go index 40ee7e3c42..1f0fe371b1 100644 --- a/server.go +++ b/server.go @@ -443,7 +443,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return err } - file, err := utils.Download(url, out) + file, err := utils.Download(url) if err != nil { return err } @@ -1248,7 +1248,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write out.Write(sf.FormatStatus("", "Downloading from %s", u)) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - resp, err = utils.Download(u.String(), out) + resp, err = utils.Download(u.String()) if err != nil { return err } diff --git a/utils/utils.go b/utils/utils.go index 0e0c231764..6864dddf82 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -45,7 +45,7 @@ func Go(f func() error) chan error { } // Request a given URL and return an io.Reader -func Download(url string, stderr io.Writer) (*http.Response, error) { +func Download(url string) (*http.Response, error) { var resp *http.Response var err error if resp, err = http.Get(url); err != nil { From be282b57d561bdb7e686ed8d4f7b6327e7a65b70 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 4 Dec 2013 11:55:42 -0800 Subject: [PATCH 146/162] fix offline image transfert --- server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server.go b/server.go index 1f0fe371b1..56e9cd222b 100644 --- a/server.go +++ b/server.go @@ -273,6 +273,9 @@ func (srv *Server) exportImage(image *Image, tempdir string) error { // temporary directory tmpImageDir := path.Join(tempdir, i.ID) if err := os.Mkdir(tmpImageDir, os.ModeDir); err != nil { + if os.IsExist(err) { + return nil + } return err } From 4bc100b494022901d4d7f51cba303ae3562b1303 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 4 Dec 2013 11:57:18 -0800 Subject: [PATCH 147/162] fix jsonmessage --- utils/jsonmessage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index eaa90a5b64..c36ec27763 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -38,11 +38,11 @@ func (p *JSONProgress) String() string { width = int(ws.Width) } - if p.Current == 0 && p.Total == 0 { + if p.Current <= 0 && p.Total <= 0 { return "" } current := HumanSize(int64(p.Current)) - if p.Total == 0 { + if p.Total <= 0 { return fmt.Sprintf("%8v", current) } total := HumanSize(int64(p.Total)) @@ -126,7 +126,7 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr, if jm.Progress != nil { jm.Progress.terminalFd = terminalFd } - if (jm.Progress != nil || jm.ProgressMessage != "") && jm.ID != "" { + if jm.Progress != nil || jm.ProgressMessage != "" { line, ok := ids[jm.ID] if !ok { line = len(ids) From 3df5d120de4f42cc208283bf78b26ca5b21c6776 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Tue, 3 Dec 2013 16:02:29 -0800 Subject: [PATCH 148/162] docs/installation: add google compute engine quickstart --- docs/sources/installation/google.rst | 65 ++++++++++++++++++++++++++++ docs/sources/installation/index.rst | 1 + 2 files changed, 66 insertions(+) create mode 100644 docs/sources/installation/google.rst diff --git a/docs/sources/installation/google.rst b/docs/sources/installation/google.rst new file mode 100644 index 0000000000..1390c684bc --- /dev/null +++ b/docs/sources/installation/google.rst @@ -0,0 +1,65 @@ +:title: Installation on Google Cloud Platform +:description: Please note this project is currently under heavy development. It should not be used in production. +:keywords: Docker, Docker documentation, installation, google, Google Compute Engine, Google Cloud Platform + +`Google Cloud Platform `_ +==================================================== + +.. include:: install_header.inc + +.. _googlequickstart: + +`Compute Engine `_ QuickStart for `Debian `_ +----------------------------------------------------------------------------------------------------------- + +1. Go to `Google Cloud Console `_ and create a new Cloud Project with billing enabled. + +2. Download and configure the `Google Cloud SDK `_ to use your project with the following commands: + +.. code-block:: bash + + $ curl https://dl.google.com/dl/cloudsdk/release/install_google_cloud_sdk.bash | bash + $ gcloud auth login + Enter a cloud project id (or leave blank to not set): + +3. Start a new instance, select a zone close to you and the desired instance size: + +.. code-block:: bash + + $ gcutil addinstance docker-playground --image=backports-debian-7 + 1: europe-west1-a + ... + 4: us-central1-b + >>> + 1: machineTypes/n1-standard-1 + ... + 12: machineTypes/g1-small + >>> + +4. Connect to the instance using SSH: + +.. code-block:: bash + + $ gcutil ssh docker-playground + docker-playground:~$ + +5. Enable IP forwarding: + +.. code-block:: bash + + docker-playground:~$ echo net.ipv4.ip_forward=1 | sudo tee /etc/sysctl.d/99-docker.conf + docker-playground:~$ sudo sysctl --system + +6. Install the latest Docker release and configure it to start when the instance boots: + +.. code-block:: bash + + docker-playground:~$ curl get.docker.io | bash + docker-playground:~$ sudo update-rc.d docker defaults + +7. Start a new container: + +.. code-block:: bash + + docker-playground:~$ sudo docker run busybox echo 'docker on GCE \o/' + docker on GCE \o/ diff --git a/docs/sources/installation/index.rst b/docs/sources/installation/index.rst index 1c9ccbe7d5..c54e6799f0 100644 --- a/docs/sources/installation/index.rst +++ b/docs/sources/installation/index.rst @@ -25,6 +25,7 @@ Contents: windows amazon rackspace + google kernel binaries security From 00030ced4bf2d0242883fae0b4ece61149f1d437 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 3 Dec 2013 15:51:43 -0700 Subject: [PATCH 149/162] Tweak Makefile for consistency and Ctrl+C-ability (also, -rm to keep it clean) --- Makefile | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 79513f0126..1518fb1331 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,26 @@ -default: build +.PHONY: all binary build default doc shell test + +DOCKER_RUN_DOCKER := docker run -rm -i -t -privileged -e TESTFLAGS -v $(CURDIR)/bundles:/go/src/github.com/dotcloud/docker/bundles docker + +default: binary + +all: build + $(DOCKER_RUN_DOCKER) hack/make.sh + +binary: build + $(DOCKER_RUN_DOCKER) hack/make.sh binary + +doc: + docker build -t docker-docs docs && docker run -p 8000:8000 docker-docs + +test: build + $(DOCKER_RUN_DOCKER) hack/make.sh test + +shell: build + $(DOCKER_RUN_DOCKER) bash build: bundles docker build -t docker . - docker run -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh binary - -doc: - cd docs && docker build -t docker-docs . && docker run -p 8000:8000 docker-docs - -test: bundles - docker run -e TESTFLAGS -privileged -v `pwd`/bundles:/go/src/github.com/dotcloud/docker/bundles docker hack/make.sh test - -shell: - docker run -privileged -i -t docker bash bundles: mkdir bundles - From 0d1506adb30ecd56f5e413c3ff376a1acfbe3697 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 4 Dec 2013 16:31:09 -0800 Subject: [PATCH 150/162] Move test cpu shares out of test start Fixes #2107 --- integration/container_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/integration/container_test.go b/integration/container_test.go index bcd00994a6..97f4cd282f 100644 --- a/integration/container_test.go +++ b/integration/container_test.go @@ -330,6 +330,36 @@ func TestCommitRun(t *testing.T) { } func TestStart(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + container, _, _ := mkContainer(runtime, []string{"-i", "_", "/bin/cat"}, t) + defer runtime.Destroy(container) + + cStdin, err := container.StdinPipe() + if err != nil { + t.Fatal(err) + } + + if err := container.Start(); err != nil { + t.Fatal(err) + } + + // Give some time to the process to start + container.WaitTimeout(500 * time.Millisecond) + + if !container.State.IsRunning() { + t.Errorf("Container should be running") + } + if err := container.Start(); err == nil { + t.Fatalf("A running container should be able to be started") + } + + // Try to avoid the timeout in destroy. Best effort, don't check error + cStdin.Close() + container.WaitTimeout(2 * time.Second) +} + +func TestCpuShares(t *testing.T) { _, err1 := os.Stat("/sys/fs/cgroup/cpuacct,cpu") _, err2 := os.Stat("/sys/fs/cgroup/cpu,cpuacct") if err1 == nil || err2 == nil { From 3ba279a3705b3417fea36e8d2eb7d54e95d60fad Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 4 Dec 2013 17:08:14 -0800 Subject: [PATCH 151/162] fix docker run -a stderr --- commands.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 96c16fa268..9df91d723e 100644 --- a/commands.go +++ b/commands.go @@ -2361,7 +2361,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea defer term.RestoreTerminal(cli.terminalFd, oldState) } - if stdout != nil { + if stdout != nil || stderr != nil { receiveStdout = utils.Go(func() (err error) { defer func() { if in != nil { @@ -2401,7 +2401,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea return nil }) - if stdout != nil { + if stdout != nil || stderr != nil { if err := <-receiveStdout; err != nil { utils.Errorf("Error receiveStdout: %s", err) return err From 97088ebef7c638f8175d7c61934642eeb2c054c4 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 4 Dec 2013 17:44:08 -0800 Subject: [PATCH 152/162] sd_notify ready status when accepting API requests --- api.go | 7 +++++-- systemd/sd_notify.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 systemd/sd_notify.go diff --git a/api.go b/api.go index 32e05a9066..0aa9695702 100644 --- a/api.go +++ b/api.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/systemd" "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" "io" @@ -1184,8 +1185,6 @@ func ServeRequest(srv *Server, apiversion float64, w http.ResponseWriter, req *h } func ListenAndServe(proto, addr string, srv *Server, logging bool) error { - log.Printf("Listening for HTTP on %s (%s)\n", addr, proto) - r, err := createRouter(srv, logging) if err != nil { return err @@ -1216,5 +1215,9 @@ func ListenAndServe(proto, addr string, srv *Server, logging bool) error { } } httpSrv := http.Server{Addr: addr, Handler: r} + + log.Printf("Listening for HTTP on %s (%s)\n", addr, proto) + // Tell the init daemon we are accepting requests + go systemd.SdNotify("READY=1") return httpSrv.Serve(l) } diff --git a/systemd/sd_notify.go b/systemd/sd_notify.go new file mode 100644 index 0000000000..1993cab9be --- /dev/null +++ b/systemd/sd_notify.go @@ -0,0 +1,33 @@ +package systemd + +import ( + "errors" + "net" + "os" +) + +var SdNotifyNoSocket = errors.New("No socket") + +// Send a message to the init daemon. It is common to ignore the error. +func SdNotify(state string) error { + socketAddr := &net.UnixAddr{ + Name: os.Getenv("NOTIFY_SOCKET"), + Net: "unixgram", + } + + if socketAddr.Name == "" { + return SdNotifyNoSocket + } + + conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr) + if err != nil { + return err + } + + _, err = conn.Write([]byte(state)) + if err != nil { + return err + } + + return nil +} From b4e21ad1da616ab46669401dc7d62a6e19b05619 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 4 Dec 2013 17:50:33 -0800 Subject: [PATCH 153/162] Add Sven Dowideit to docs maintainers --- docs/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index 8ef58199e7..5f83c635d8 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,3 +1,4 @@ Andy Rothfusz (@metalivedev) Ken Cochrane (@kencochrane) James Turnbull (@jamesturnbull) +Sven Dowideit (@SvenDowideit) From af020e2d67256d386e43741071dc4855735d92a3 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 5 Dec 2013 14:14:08 +1000 Subject: [PATCH 154/162] I was reading the doc, and noticed that some examples didnt use $ sudo docker, so this makes it a little more consistent --- docs/sources/commandline/cli.rst | 52 ++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index d7bb03b954..ba3c44c29d 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -143,7 +143,7 @@ Examples: .. code-block:: bash - sudo docker build . + $ sudo docker build . Uploading context 10240 bytes Step 1 : FROM busybox Pulling repository busybox @@ -183,7 +183,7 @@ message. .. code-block:: bash - sudo docker build -t vieux/apache:2.0 . + $ sudo docker build -t vieux/apache:2.0 . This will build like the previous example, but it will then tag the resulting image. The repository name will be ``vieux/apache`` and the @@ -192,7 +192,7 @@ tag will be ``2.0`` .. code-block:: bash - sudo docker build - < Dockerfile + $ sudo docker build - < Dockerfile This will read a ``Dockerfile`` from *stdin* without context. Due to the lack of a context, no contents of any local directory will be sent @@ -201,7 +201,7 @@ to the ``docker`` daemon. Since there is no context, a Dockerfile .. code-block:: bash - sudo docker build github.com/creack/docker-firefox + $ sudo docker build github.com/creack/docker-firefox This will clone the Github repository and use the cloned repository as context. The ``Dockerfile`` at the root of the repository is used as @@ -230,7 +230,7 @@ Simple commit of an existing container .. code-block:: bash - $ docker ps + $ sudo docker ps ID IMAGE COMMAND CREATED STATUS PORTS c3f279d17e0a ubuntu:12.04 /bin/bash 7 days ago Up 25 hours 197387f1b436 ubuntu:12.04 /bin/bash 7 days ago Up 25 hours @@ -678,7 +678,7 @@ fairly straightforward manner. .. code-block:: bash - docker inspect -format='{{.NetworkSettings.IPAddress}}' $INSTANCE_ID + $ sudo docker inspect -format='{{.NetworkSettings.IPAddress}}' $INSTANCE_ID List All Port Bindings ...................... @@ -688,13 +688,11 @@ text output: .. code-block:: bash - docker inspect -format='{{range $p, $conf := .NetworkSettings.Ports}} {{$p}} -> {{(index $conf 0).HostPort}} {{end}}' $INSTANCE_ID + $ sudo docker inspect -format='{{range $p, $conf := .NetworkSettings.Ports}} {{$p}} -> {{(index $conf 0).HostPort}} {{end}}' $INSTANCE_ID Find a Specific Port Mapping ............................ -.. code-block:: bash - The ``.Field`` syntax doesn't work when the field name begins with a number, but the template language's ``index`` function does. The ``.NetworkSettings.Ports`` section contains a map of the internal port @@ -705,7 +703,7 @@ we ask for the ``HostPort`` field to get the public address. .. code-block:: bash - docker inspect -format='{{(index (index .NetworkSettings.Ports "8787/tcp") 0).HostPort}}' $INSTANCE_ID + $ sudo docker inspect -format='{{(index (index .NetworkSettings.Ports "8787/tcp") 0).HostPort}}' $INSTANCE_ID .. _cli_kill: @@ -858,7 +856,7 @@ Examples: .. code-block:: bash - $ docker rm /redis + $ sudo docker rm /redis /redis @@ -867,7 +865,7 @@ This will remove the container referenced under the link ``/redis``. .. code-block:: bash - $ docker rm -link /webapp/redis + $ sudo docker rm -link /webapp/redis /webapp/redis @@ -876,7 +874,7 @@ network communication. .. code-block:: bash - $ docker rm `docker ps -a -q` + $ sudo docker rm `docker ps -a -q` This command will delete all stopped containers. The command ``docker ps -a -q`` will return all @@ -943,7 +941,7 @@ Examples: .. code-block:: bash - sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" + $ sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" This will create a container and print "test" to the console. The ``cidfile`` flag makes docker attempt to create a new file and write the @@ -952,7 +950,10 @@ error. Docker will close this file when docker run exits. .. code-block:: bash - docker run mount -t tmpfs none /var/spool/squid + $ sudo docker run -t -i -rm ubuntu bash + root@bc338942ef20:/# mount -t tmpfs none /mnt + mount: permission denied + This will *not* work, because by default, most potentially dangerous kernel capabilities are dropped; including ``cap_sys_admin`` (which is @@ -961,7 +962,12 @@ allow it to run: .. code-block:: bash - docker run -privileged mount -t tmpfs none /var/spool/squid + $ sudo docker run -privileged ubuntu bash + root@50e3f57e16e6:/# mount -t tmpfs none /mnt + root@50e3f57e16e6:/# df -h + Filesystem Size Used Avail Use% Mounted on + none 1.9G 0 1.9G 0% /mnt + The ``-privileged`` flag gives *all* capabilities to the container, and it also lifts all the limitations enforced by the ``device`` @@ -971,7 +977,7 @@ use-cases, like running Docker within Docker. .. code-block:: bash - docker run -w /path/to/dir/ -i -t ubuntu pwd + $ sudo docker run -w /path/to/dir/ -i -t ubuntu pwd The ``-w`` lets the command being executed inside directory given, here /path/to/dir/. If the path does not exists it is created inside the @@ -979,7 +985,7 @@ container. .. code-block:: bash - docker run -v `pwd`:`pwd` -w `pwd` -i -t ubuntu pwd + $ sudo docker run -v `pwd`:`pwd` -w `pwd` -i -t ubuntu pwd The ``-v`` flag mounts the current working directory into the container. The ``-w`` lets the command being executed inside the current @@ -989,7 +995,7 @@ using the container, but inside the current working directory. .. code-block:: bash - docker run -p 127.0.0.1:80:8080 ubuntu bash + $ sudo docker run -p 127.0.0.1:80:8080 ubuntu bash This binds port ``8080`` of the container to port ``80`` on 127.0.0.1 of the host machine. :ref:`port_redirection` explains in detail how to manipulate ports @@ -997,7 +1003,7 @@ in Docker. .. code-block:: bash - docker run -expose 80 ubuntu bash + $ sudo docker run -expose 80 ubuntu bash This exposes port ``80`` of the container for use within a link without publishing the port to the host system's interfaces. :ref:`port_redirection` @@ -1005,14 +1011,14 @@ explains in detail how to manipulate ports in Docker. .. code-block:: bash - docker run -name console -t -i ubuntu bash + $ sudo docker run -name console -t -i ubuntu bash This will create and run a new container with the container name being ``console``. .. code-block:: bash - docker run -link /redis:redis -name console ubuntu bash + $ sudo docker run -link /redis:redis -name console ubuntu bash The ``-link`` flag will link the container named ``/redis`` into the newly created container with the alias ``redis``. The new container @@ -1022,7 +1028,7 @@ to the newly created container. .. code-block:: bash - docker run -volumes-from 777f7dc92da7,ba8c0c54f0f2:ro -i -t ubuntu pwd + $ sudo docker run -volumes-from 777f7dc92da7,ba8c0c54f0f2:ro -i -t ubuntu pwd The ``-volumes-from`` flag mounts all the defined volumes from the refrence containers. Containers can be specified by a comma seperated From 909da5d5249880b02bad5fb893d12fe2a3fdd942 Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Wed, 4 Dec 2013 21:08:44 -0800 Subject: [PATCH 155/162] - Added link to the source code repo on github --- docs/sources/api/registry_index_spec.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/api/registry_index_spec.rst b/docs/sources/api/registry_index_spec.rst index 58b205672c..449569414e 100644 --- a/docs/sources/api/registry_index_spec.rst +++ b/docs/sources/api/registry_index_spec.rst @@ -37,6 +37,7 @@ We expect that there will be only one instance of the index, run and managed by - It delegates authentication and authorization to the Index Auth service using tokens - It supports different storage backends (S3, cloud files, local FS) - It doesn’t have a local database +- `Source Code `_ We expect that there will be multiple registries out there. To help to grasp the context, here are some examples of registries: From e9bf971e69800f1189dd240d95972f5f0dcc3bec Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 5 Dec 2013 21:48:35 +1000 Subject: [PATCH 156/162] use the Makefile in the dev environment documentation - its way less typing, and fewer typing mistakes --- docs/sources/contributing/devenvironment.rst | 31 +++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/sources/contributing/devenvironment.rst b/docs/sources/contributing/devenvironment.rst index fdbc6d7e1f..c152ff9ce5 100644 --- a/docs/sources/contributing/devenvironment.rst +++ b/docs/sources/contributing/devenvironment.rst @@ -42,9 +42,7 @@ This following command will build a development environment using the Dockerfile .. code-block:: bash - sudo docker build -t docker . - - + sudo make build If the build is successful, congratulations! You have produced a clean build of docker, neatly encapsulated in a standard build environment. @@ -56,7 +54,7 @@ To create the Docker binary, run this command: .. code-block:: bash - sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh binary + sudo make binary This will create the Docker binary in ``./bundles/-dev/binary/`` @@ -68,10 +66,15 @@ To execute the test cases, run this command: .. code-block:: bash - sudo docker run -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test + sudo make test -Note: if you're running the tests in vagrant, you need to specify a dns entry in the command: `-dns 8.8.8.8` +Note: if you're running the tests in vagrant, you need to specify a dns entry in +the command (either edit the Makefile, or run the step manually): + +.. code-block:: bash + + sudo docker run -dns 8.8.8.8 -privileged -v `pwd`:/go/src/github.com/dotcloud/docker docker hack/make.sh test If the test are successful then the tail of the output should look something like this @@ -113,11 +116,23 @@ You can run an interactive session in the newly built container: .. code-block:: bash - sudo docker run -privileged -i -t docker bash + sudo make shell - # type 'exit' to exit + # type 'exit' or Ctrl-D to exit +Extra Step: Build and view the Documenation +------------------------------------------- + +If you want to read the documentation from a local website, or are making changes +to it, you can build the documentation and then serve it by: + +.. code-block:: bash + + sudo make doc + # when its done, you can point your browser to http://yourdockerhost:8000 + # type Ctrl-C to exit + .. note:: The binary is available outside the container in the directory ``./bundles/-dev/binary/``. You can swap your host docker executable with this binary for live testing - for example, on ubuntu: ``sudo service docker stop ; sudo cp $(which docker) $(which docker)_ ; sudo cp ./bundles/-dev/binary/docker--dev $(which docker);sudo service docker start``. From 7a94cdf8edd917899a38a58917cd4439652047a1 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 2 Dec 2013 11:03:21 -0600 Subject: [PATCH 157/162] create the bridge device with ioctl On RHEL 6, creation of a bridge device with netlink fails. Use the more backward-compatible ioctl instead. This fixes networking on RHEL 6. --- network.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/network.go b/network.go index fc76bc3421..a230356a7d 100644 --- a/network.go +++ b/network.go @@ -12,6 +12,8 @@ import ( "net" "strconv" "sync" + "syscall" + "unsafe" ) const ( @@ -19,6 +21,7 @@ const ( DisableNetworkBridge = "none" portRangeStart = 49153 portRangeEnd = 65535 + siocBRADDBR = 0x89a0 ) // Calculates the first and last IP addresses in an IPNet @@ -149,8 +152,8 @@ func CreateBridgeIface(config *DaemonConfig) error { } utils.Debugf("Creating bridge %s with network %s", config.BridgeIface, ifaceAddr) - if err := netlink.NetworkLinkAdd(config.BridgeIface, "bridge"); err != nil { - return fmt.Errorf("Error creating bridge: %s", err) + if err := createBridgeIface(config.BridgeIface); err != nil { + return err } iface, err := net.InterfaceByName(config.BridgeIface) if err != nil { @@ -170,6 +173,26 @@ func CreateBridgeIface(config *DaemonConfig) error { return nil } +// Create the actual bridge device. This is more backward-compatible than +// netlink.NetworkLinkAdd and works on RHEL 6. +func createBridgeIface(name string) error { + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_IP) + if err != nil { + return fmt.Errorf("Error creating bridge creation socket: %s", err) + } + defer syscall.Close(s) + + nameBytePtr, err := syscall.BytePtrFromString(name) + if err != nil { + return fmt.Errorf("Error converting bridge name %s to byte array: %s", name, err) + } + + if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(s), siocBRADDBR, uintptr(unsafe.Pointer(nameBytePtr))); err != 0 { + return fmt.Errorf("Error creating bridge: %s", err) + } + return nil +} + // Return the IPv4 address of a network interface func getIfaceAddr(name string) (net.Addr, error) { iface, err := net.InterfaceByName(name) From 697707e4afe6f1e7e5e33c24ada2f1f2af279142 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 5 Dec 2013 14:03:23 -0800 Subject: [PATCH 158/162] Save layersize on pull Do not display size and virtual size on the cli. Only display virtual size on the cli --- commands.go | 9 ++----- docs/sources/commandline/cli.rst | 44 ++++++++++++++++---------------- graph.go | 22 +++++++++++++--- image.go | 32 ++++++++++++----------- integration/commands_test.go | 2 +- 5 files changed, 60 insertions(+), 49 deletions(-) diff --git a/commands.go b/commands.go index 9df91d723e..b7e7c08cac 100644 --- a/commands.go +++ b/commands.go @@ -1200,7 +1200,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tSIZE") + fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE") } for _, out := range outs { @@ -1213,12 +1213,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { } if !*quiet { - fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t", repo, tag, out.ID, utils.HumanDuration(time.Now().UTC().Sub(time.Unix(out.Created, 0)))) - if out.VirtualSize > 0 { - fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.Size), utils.HumanSize(out.VirtualSize)) - } else { - fmt.Fprintf(w, "%s\n", utils.HumanSize(out.Size)) - } + fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, out.ID, utils.HumanDuration(time.Now().UTC().Sub(time.Unix(out.Created, 0))), utils.HumanSize(out.VirtualSize)) } else { fmt.Fprintln(w, out.ID) } diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index d7bb03b954..a2132476c9 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -237,8 +237,8 @@ Simple commit of an existing container $ docker commit c3f279d17e0a SvenDowideit/testimage:version3 f5283438590d $ docker images | head - REPOSITORY TAG ID CREATED SIZE - SvenDowideit/testimage version3 f5283438590d 16 seconds ago 204.2 MB (virtual 335.7 MB) + REPOSITORY TAG ID CREATED VIRTUAL SIZE + SvenDowideit/testimage version3 f5283438590d 16 seconds ago 335.7 MB Full -run example @@ -481,16 +481,16 @@ Listing the most recently created images .. code-block:: bash $ sudo docker images | head - REPOSITORY TAG IMAGE ID CREATED SIZE - 77af4d6b9913 19 hours ago 30.53 MB (virtual 1.089 GB) - committest latest b6fa739cedf5 19 hours ago 30.53 MB (virtual 1.089 GB) - 78a85c484f71 19 hours ago 30.53 MB (virtual 1.089 GB) - docker latest 30557a29d5ab 20 hours ago 30.53 MB (virtual 1.089 GB) - 0124422dd9f9 20 hours ago 30.53 MB (virtual 1.089 GB) - 18ad6fad3402 22 hours ago 23.68 MB (virtual 1.082 GB) - f9f1e26352f0 23 hours ago 30.46 MB (virtual 1.089 GB) - tryout latest 2629d1fa0b81 23 hours ago 16.4 kB (virtual 131.5 MB) - 5ed6274db6ce 24 hours ago 30.44 MB (virtual 1.089 GB) + REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE + 77af4d6b9913 19 hours ago 1.089 GB + committest latest b6fa739cedf5 19 hours ago 1.089 GB + 78a85c484f71 19 hours ago 1.089 GB + docker latest 30557a29d5ab 20 hours ago 1.089 GB + 0124422dd9f9 20 hours ago 1.089 GB + 18ad6fad3402 22 hours ago 1.082 GB + f9f1e26352f0 23 hours ago 1.089 GB + tryout latest 2629d1fa0b81 23 hours ago 131.5 MB + 5ed6274db6ce 24 hours ago 1.089 GB Listing the full length image IDs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -498,16 +498,16 @@ Listing the full length image IDs .. code-block:: bash $ sudo docker images -notrunc | head - REPOSITORY TAG IMAGE ID CREATED SIZE - 77af4d6b9913e693e8d0b4b294fa62ade6054e6b2f1ffb617ac955dd63fb0182 19 hours ago 30.53 MB (virtual 1.089 GB) - committest latest b6fa739cedf5ea12a620a439402b6004d057da800f91c7524b5086a5e4749c9f 19 hours ago 30.53 MB (virtual 1.089 GB) - 78a85c484f71509adeaace20e72e941f6bdd2b25b4c75da8693efd9f61a37921 19 hours ago 30.53 MB (virtual 1.089 GB) - docker latest 30557a29d5abc51e5f1d5b472e79b7e296f595abcf19fe6b9199dbbc809c6ff4 20 hours ago 30.53 MB (virtual 1.089 GB) - 0124422dd9f9cf7ef15c0617cda3931ee68346455441d66ab8bdc5b05e9fdce5 20 hours ago 30.53 MB (virtual 1.089 GB) - 18ad6fad340262ac2a636efd98a6d1f0ea775ae3d45240d3418466495a19a81b 22 hours ago 23.68 MB (virtual 1.082 GB) - f9f1e26352f0a3ba6a0ff68167559f64f3e21ff7ada60366e2d44a04befd1d3a 23 hours ago 30.46 MB (virtual 1.089 GB) - tryout latest 2629d1fa0b81b222fca63371ca16cbf6a0772d07759ff80e8d1369b926940074 23 hours ago 16.4 kB (virtual 131.5 MB) - 5ed6274db6ceb2397844896966ea239290555e74ef307030ebb01ff91b1914df 24 hours ago 30.44 MB (virtual 1.089 GB) + REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE + 77af4d6b9913e693e8d0b4b294fa62ade6054e6b2f1ffb617ac955dd63fb0182 19 hours ago 1.089 GB + committest latest b6fa739cedf5ea12a620a439402b6004d057da800f91c7524b5086a5e4749c9f 19 hours ago 1.089 GB + 78a85c484f71509adeaace20e72e941f6bdd2b25b4c75da8693efd9f61a37921 19 hours ago 1.089 GB + docker latest 30557a29d5abc51e5f1d5b472e79b7e296f595abcf19fe6b9199dbbc809c6ff4 20 hours ago 1.089 GB + 0124422dd9f9cf7ef15c0617cda3931ee68346455441d66ab8bdc5b05e9fdce5 20 hours ago 1.089 GB + 18ad6fad340262ac2a636efd98a6d1f0ea775ae3d45240d3418466495a19a81b 22 hours ago 1.082 GB + f9f1e26352f0a3ba6a0ff68167559f64f3e21ff7ada60366e2d44a04befd1d3a 23 hours ago 1.089 GB + tryout latest 2629d1fa0b81b222fca63371ca16cbf6a0772d07759ff80e8d1369b926940074 23 hours ago 131.5 MB + 5ed6274db6ceb2397844896966ea239290555e74ef307030ebb01ff91b1914df 24 hours ago 1.089 GB Displaying images visually ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/graph.go b/graph.go index 19c668830e..82128bfe15 100644 --- a/graph.go +++ b/graph.go @@ -94,11 +94,25 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) } img.graph = graph - if img.Size == 0 { - size, err := utils.TreeSize(rootfs) - if err != nil { - return nil, fmt.Errorf("Error computing size of rootfs %s: %s", img.ID, err) + + if img.Size < 0 { + var size int64 + if img.Parent == "" { + if size, err = utils.TreeSize(rootfs); err != nil { + return nil, err + } + } else { + parentFs, err := graph.driver.Get(img.Parent) + if err != nil { + return nil, err + } + changes, err := archive.ChangesDirs(rootfs, parentFs) + if err != nil { + return nil, err + } + size = archive.ChangesSize(rootfs, changes) } + img.Size = size if err := img.SaveSize(graph.imageRoot(id)); err != nil { return nil, err diff --git a/image.go b/image.go index 5a0b043fc7..0f07a74ee8 100644 --- a/image.go +++ b/image.go @@ -51,6 +51,9 @@ func LoadImage(root string) (*Image, error) { if !os.IsNotExist(err) { return nil, err } + // If the layersize file does not exist then set the size to a negative number + // because a layer size of 0 (zero) is valid + img.Size = -1 } else { size, err := strconv.Atoi(string(buf)) if err != nil { @@ -104,30 +107,29 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root, la if err != nil { return err } - if size = archive.ChangesSize(layer, changes); err != nil { - return err - } + size = archive.ChangesSize(layer, changes) } } } - // If raw json is provided, then use it - if jsonData != nil { - return ioutil.WriteFile(jsonPath(root), jsonData, 0600) - } - // Otherwise, unmarshal the image - if jsonData, err = json.Marshal(img); err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } - img.Size = size if err := img.SaveSize(root); err != nil { return err } + // If raw json is provided, then use it + if jsonData != nil { + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } else { + if jsonData, err = json.Marshal(img); err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } return nil } diff --git a/integration/commands_test.go b/integration/commands_test.go index 129a1575f8..16e2b3fb02 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -871,7 +871,7 @@ func TestImagesTree(t *testing.T) { "(?m) └─[0-9a-f]+.*", "(?m) └─[0-9a-f]+.*", "(?m) └─[0-9a-f]+.*", - fmt.Sprintf("(?m)^ └─%s Size: \\d+.\\d+ MB \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)), + fmt.Sprintf("(?m)^ └─%s Size: \\d+ B \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)), } compiledRegexps := []*regexp.Regexp{} From a96bf7439774d905f72e835218c72a5b45da37c0 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Thu, 5 Dec 2013 14:34:10 -0800 Subject: [PATCH 159/162] include Red Hat Enterprise, should have part of #3011 oops, forgot the index! --- docs/sources/installation/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sources/installation/index.rst b/docs/sources/installation/index.rst index c54e6799f0..b2882a5cb3 100644 --- a/docs/sources/installation/index.rst +++ b/docs/sources/installation/index.rst @@ -18,6 +18,7 @@ Contents: :maxdepth: 1 ubuntulinux + rhel fedora archlinux gentoolinux From fb810b54ff404e81bd2486a944e24583a55fe8bf Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 5 Dec 2013 14:41:56 -0800 Subject: [PATCH 160/162] wait on pull from another client --- server.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index deda6fb663..49b25f7098 100644 --- a/server.go +++ b/server.go @@ -877,8 +877,14 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName } // ensure no two downloads of the same image happen at the same time - if _, err := srv.poolAdd("pull", "img:"+img.ID); err != nil { - utils.Errorf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) + if c, err := srv.poolAdd("pull", "img:"+img.ID); err != nil { + if c != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) + <-c + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) + } else { + utils.Errorf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) + } if parallel { errors <- nil } From 7bf3a07371bb0baed21b59d60835ed2f4877b571 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 5 Dec 2013 15:22:21 -0800 Subject: [PATCH 161/162] If container does not exist try to remove the name and continue --- runtime.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/runtime.go b/runtime.go index 2e2492a6e1..fbf0facd7d 100644 --- a/runtime.go +++ b/runtime.go @@ -425,12 +425,26 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin // Set the enitity in the graph using the default name specified if _, err := runtime.containerGraph.Set(name, id); err != nil { - if strings.HasSuffix(err.Error(), "name are not unique") { - conflictingContainer, _ := runtime.GetByName(name) - nameAsKnownByUser := strings.TrimPrefix(name, "/") - return nil, nil, fmt.Errorf("Conflict, The name %s is already assigned to %s. You have to delete (or rename) that container to be able to assign %s to a container again.", nameAsKnownByUser, utils.TruncateID(conflictingContainer.ID), nameAsKnownByUser) + if !strings.HasSuffix(err.Error(), "name are not unique") { + return nil, nil, err + } + + conflictingContainer, err := runtime.GetByName(name) + if err != nil { + if strings.Contains(err.Error(), "Could not find entity") { + return nil, nil, err + } + + // Remove name and continue starting the container + if err := runtime.containerGraph.Delete(name); err != nil { + return nil, nil, err + } + } else { + nameAsKnownByUser := strings.TrimPrefix(name, "/") + return nil, nil, fmt.Errorf( + "Conflict, The name %s is already assigned to %s. You have to delete (or rename) that container to be able to assign %s to a container again.", nameAsKnownByUser, + utils.TruncateID(conflictingContainer.ID), nameAsKnownByUser) } - return nil, nil, err } // Generate default hostname From e39d35dedad1db952c30958303902b4c96e1f406 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 5 Dec 2013 16:32:02 -0800 Subject: [PATCH 162/162] Bump to 0.7.1 --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd91eaddc..86e3b88484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # Changelog +## 0.7.1 (2013-12-05) + +#### Documentation + ++ Add @SvenDowideit as documentation maintainer ++ Add links example ++ Add documentation regarding ambassador pattern ++ Add Google Cloud Platform docs ++ Add dockerfile best practices +* Update doc for RHEL +* Update doc for registry +* Update Postgres examples +* Update doc for Ubuntu install +* Improve remote api doc + +#### Runtime + ++ Add hostconfig to docker inspect ++ Implement `docker log -f` to stream logs ++ Add env variable to disable kernel version warning ++ Add -format to `docker inspect` ++ Support bind-mount for files +- Fix bridge creation on RHEL +- Fix image size calculation +- Make sure iptables are called even if the bridge already exists +- Fix issue with stderr only attach +- Remove init layer when destroying a container +- Fix same port binding on different interfaces +- `docker build` now returns the correct exit code +- Fix `docker port` to display correct port +- `docker build` now check that the dockerfile exists client side +- `docker attach` now returns the correct exit code +- Remove the name entry when the container does not exist + +#### Registry + +* Improve progress bars, add ETA for downloads +* Simultaneous pulls now waits for the first to finish instead of failing +- Tag only the top-layer image when pushing to registry +- Fix issue with offline image transfer +- Fix issue preventing using ':' in password for registry + +#### Other + ++ Add pprof handler for debug ++ Create a Makefile +* Use stdlib tar that now includes fix +* Improve make.sh test script +* Handle SIGQUIT on the daemon +* Disable verbose during tests +* Upgrade to go1.2 for official build +* Improve unit tests +* The test suite now runs all tests even if one fails +* Refactor C in Go (Devmapper) +- Fix OSX compilation + ## 0.7.0 (2013-11-25) #### Notable features since 0.6.0 diff --git a/VERSION b/VERSION index e1bde8025c..39e898a4f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.0-dev +0.7.1