From 58f3b576c915be46b7fe828e4fd47e86742b02d9 Mon Sep 17 00:00:00 2001 From: Ben Hamill Date: Fri, 17 Apr 2015 15:47:28 -0500 Subject: [PATCH 001/337] Replace - with _ in completion identifiers. Signed-off-by: Ben Hamill --- contrib/completion/bash/docker-compose | 100 ++++++++++++------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index b785a99253..a8a5084dab 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -20,7 +20,7 @@ # For compatibility reasons, Compose and therefore its completion supports several # stack compositon files as listed here, in descending priority. # Support for these filenames might be dropped in some future version. -__docker-compose_compose_file() { +__docker_compose_compose_file() { local file for file in docker-compose.y{,a}ml fig.y{,a}ml ; do [ -e $file ] && { @@ -32,34 +32,34 @@ __docker-compose_compose_file() { } # Extracts all service names from the compose file. -___docker-compose_all_services_in_compose_file() { - awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null +___docker_compose_all_services_in_compose_file() { + awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null } # All services, even those without an existing container -__docker-compose_services_all() { - COMPREPLY=( $(compgen -W "$(___docker-compose_all_services_in_compose_file)" -- "$cur") ) +__docker_compose_services_all() { + COMPREPLY=( $(compgen -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) } # All services that have an entry with the given key in their compose_file section -___docker-compose_services_with_key() { +___docker_compose_services_with_key() { # flatten sections to one line, then filter lines containing the key and return section name. - awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker-compose_compose_file)}" | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' + awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker_compose_compose_file)}" | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' } # All services that are defined by a Dockerfile reference -__docker-compose_services_from_build() { - COMPREPLY=( $(compgen -W "$(___docker-compose_services_with_key build)" -- "$cur") ) +__docker_compose_services_from_build() { + COMPREPLY=( $(compgen -W "$(___docker_compose_services_with_key build)" -- "$cur") ) } # All services that are defined by an image -__docker-compose_services_from_image() { - COMPREPLY=( $(compgen -W "$(___docker-compose_services_with_key image)" -- "$cur") ) +__docker_compose_services_from_image() { + COMPREPLY=( $(compgen -W "$(___docker_compose_services_with_key image)" -- "$cur") ) } # The services for which containers have been created, optionally filtered # by a boolean expression passed in as argument. -__docker-compose_services_with() { +__docker_compose_services_with() { local containers names containers="$(docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)" names=( $(docker 2>/dev/null inspect --format "{{if ${1:-true}}} {{ .Name }} {{end}}" $containers) ) @@ -69,29 +69,29 @@ __docker-compose_services_with() { } # The services for which at least one running container exists -__docker-compose_services_running() { - __docker-compose_services_with '.State.Running' +__docker_compose_services_running() { + __docker_compose_services_with '.State.Running' } # The services for which at least one stopped container exists -__docker-compose_services_stopped() { - __docker-compose_services_with 'not .State.Running' +__docker_compose_services_stopped() { + __docker_compose_services_with 'not .State.Running' } -_docker-compose_build() { +_docker_compose_build() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--help --no-cache" -- "$cur" ) ) ;; *) - __docker-compose_services_from_build + __docker_compose_services_from_build ;; esac } -_docker-compose_docker-compose() { +_docker_compose_docker_compose() { case "$prev" in --file|-f) _filedir "y?(a)ml" @@ -113,12 +113,12 @@ _docker-compose_docker-compose() { } -_docker-compose_help() { +_docker_compose_help() { COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) } -_docker-compose_kill() { +_docker_compose_kill() { case "$prev" in -s) COMPREPLY=( $( compgen -W "SIGHUP SIGINT SIGKILL SIGUSR1 SIGUSR2" -- "$(echo $cur | tr '[:lower:]' '[:upper:]')" ) ) @@ -131,25 +131,25 @@ _docker-compose_kill() { COMPREPLY=( $( compgen -W "--help -s" -- "$cur" ) ) ;; *) - __docker-compose_services_running + __docker_compose_services_running ;; esac } -_docker-compose_logs() { +_docker_compose_logs() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--help --no-color" -- "$cur" ) ) ;; *) - __docker-compose_services_all + __docker_compose_services_all ;; esac } -_docker-compose_migrate-to-labels() { +_docker_compose_migrate_to_labels() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) @@ -158,7 +158,7 @@ _docker-compose_migrate-to-labels() { } -_docker-compose_port() { +_docker_compose_port() { case "$prev" in --protocol) COMPREPLY=( $( compgen -W "tcp udp" -- "$cur" ) ) @@ -174,37 +174,37 @@ _docker-compose_port() { COMPREPLY=( $( compgen -W "--help --index --protocol" -- "$cur" ) ) ;; *) - __docker-compose_services_all + __docker_compose_services_all ;; esac } -_docker-compose_ps() { +_docker_compose_ps() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--help -q" -- "$cur" ) ) ;; *) - __docker-compose_services_all + __docker_compose_services_all ;; esac } -_docker-compose_pull() { +_docker_compose_pull() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--allow-insecure-ssl --help" -- "$cur" ) ) ;; *) - __docker-compose_services_from_image + __docker_compose_services_from_image ;; esac } -_docker-compose_restart() { +_docker_compose_restart() { case "$prev" in -t | --timeout) return @@ -216,25 +216,25 @@ _docker-compose_restart() { COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) - __docker-compose_services_running + __docker_compose_services_running ;; esac } -_docker-compose_rm() { +_docker_compose_rm() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--force -f --help -v" -- "$cur" ) ) ;; *) - __docker-compose_services_stopped + __docker_compose_services_stopped ;; esac } -_docker-compose_run() { +_docker_compose_run() { case "$prev" in -e) COMPREPLY=( $( compgen -e -- "$cur" ) ) @@ -251,19 +251,19 @@ _docker-compose_run() { COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --entrypoint -e --help --no-deps --rm --service-ports -T --user -u" -- "$cur" ) ) ;; *) - __docker-compose_services_all + __docker_compose_services_all ;; esac } -_docker-compose_scale() { +_docker_compose_scale() { case "$prev" in =) COMPREPLY=("$cur") ;; *) - COMPREPLY=( $(compgen -S "=" -W "$(___docker-compose_all_services_in_compose_file)" -- "$cur") ) + COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) compopt -o nospace ;; esac @@ -276,19 +276,19 @@ _docker-compose_scale() { } -_docker-compose_start() { +_docker_compose_start() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; *) - __docker-compose_services_stopped + __docker_compose_services_stopped ;; esac } -_docker-compose_stop() { +_docker_compose_stop() { case "$prev" in -t | --timeout) return @@ -300,13 +300,13 @@ _docker-compose_stop() { COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) - __docker-compose_services_running + __docker_compose_services_running ;; esac } -_docker-compose_up() { +_docker_compose_up() { case "$prev" in -t | --timeout) return @@ -318,13 +318,13 @@ _docker-compose_up() { COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --help --no-build --no-color --no-deps --no-recreate --timeout -t --x-smart-recreate" -- "$cur" ) ) ;; *) - __docker-compose_services_all + __docker_compose_services_all ;; esac } -_docker-compose_version() { +_docker_compose_version() { case "$cur" in -*) COMPREPLY=( $( compgen -W "--short" -- "$cur" ) ) @@ -333,7 +333,7 @@ _docker-compose_version() { } -_docker-compose() { +_docker_compose() { local previous_extglob_setting=$(shopt -p extglob) shopt -s extglob @@ -362,7 +362,7 @@ _docker-compose() { # search subcommand and invoke its handler. # special treatment of some top-level options - local command='docker-compose' + local command='docker_compose' local counter=1 local compose_file compose_project while [ $counter -lt $cword ]; do @@ -385,11 +385,11 @@ _docker-compose() { (( counter++ )) done - local completions_func=_docker-compose_${command} + local completions_func=_docker_compose_${command//-/_} declare -F $completions_func >/dev/null && $completions_func eval "$previous_extglob_setting" return 0 } -complete -F _docker-compose docker-compose +complete -F _docker_compose docker-compose From aab6df6ba4e4b13f56effbfc632c2030b89232a9 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 28 May 2015 19:02:19 +0100 Subject: [PATCH 002/337] Add test coverage support Prints out results on console and puts HTML report in `coverage-html`. Signed-off-by: Ben Firshman --- .dockerignore | 1 + .gitignore | 1 + requirements-dev.txt | 1 + script/test | 3 +++ script/test-versions | 3 ++- 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index a03616e534..b85b7e5d86 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ build dist venv +coverage-html diff --git a/.gitignore b/.gitignore index da7fe7fa47..52a78bd974 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ /docs/_site /venv docker-compose.spec +coverage-html diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b529623fb..c5d9c10645 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c unittest2==0.8.0 flake8==2.3.0 pep8==1.6.1 +coverage==3.7.1 diff --git a/script/test b/script/test index 625af09b35..adf3fb1bab 100755 --- a/script/test +++ b/script/test @@ -5,6 +5,8 @@ set -ex TAG="docker-compose:$(git rev-parse --short HEAD)" +rm -rf coverage-html + docker build -t "$TAG" . docker run \ --rm \ @@ -12,6 +14,7 @@ docker run \ -e DOCKER_VERSIONS \ -e "TAG=$TAG" \ -e "affinity:image==$TAG" \ + -e "COVERAGE_DIR=$(pwd)/coverage-html" \ --entrypoint="script/test-versions" \ "$TAG" \ "$@" diff --git a/script/test-versions b/script/test-versions index 7f1a14a9b1..9e81a515d9 100755 --- a/script/test-versions +++ b/script/test-versions @@ -19,8 +19,9 @@ for version in $DOCKER_VERSIONS; do --rm \ --privileged \ --volume="/var/lib/docker" \ + --volume="${COVERAGE_DIR:-$(pwd)/coverage-html}:/code/coverage-html" \ -e "DOCKER_VERSION=$version" \ --entrypoint="script/dind" \ "$TAG" \ - script/wrapdocker nosetests "$@" + script/wrapdocker nosetests --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html "$@" done From 7eabc06df5ca4a1c2ad372ee8e87012de5429f05 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Sun, 19 Jul 2015 14:39:25 -0700 Subject: [PATCH 003/337] Updating build so that contributors can build public docs Changed base image Signed-off-by: Mary Anthony --- docs/Dockerfile | 16 +++++----- docs/pre-process.sh | 61 +++++++++++++++++++++++++++++++++++++++ docs/reference/build.md | 1 + docs/reference/help.md | 1 + docs/reference/kill.md | 1 + docs/reference/logs.md | 1 + docs/reference/port.md | 1 + docs/reference/ps.md | 1 + docs/reference/pull.md | 1 + docs/reference/restart.md | 1 + docs/reference/rm.md | 1 + docs/reference/run.md | 1 + docs/reference/start.md | 1 + docs/reference/stop.md | 1 + docs/reference/up.md | 1 + 15 files changed, 83 insertions(+), 7 deletions(-) create mode 100755 docs/pre-process.sh diff --git a/docs/Dockerfile b/docs/Dockerfile index d6864c2d66..d9add75c15 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -6,6 +6,14 @@ COPY . /src COPY . /docs/content/compose/ +RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/docker +RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm +RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine +RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content/registry +RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/tutorials +RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content + + # Sed to process GitHub Markdown # 1-2 Remove comment code from metadata block # 3 Change ](/word to ](/project/ in links @@ -15,10 +23,4 @@ COPY . /docs/content/compose/ # 7 Change ](../../ to ](/project/ --> not implemented # # -RUN find /docs/content/compose -type f -name "*.md" -exec sed -i.old \ - -e '/^/g' \ - -e '/^/g' \ - -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/compose\//g' \ - -e 's/\(\][(]\)\([A-z].*\)\(\.md\)/\1\/compose\/\2/g' \ - -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ - -e 's/\(\][(]\)\(\.\.\/\)/\1\/compose\//g' {} \; +RUN /src/pre-process.sh /docs diff --git a/docs/pre-process.sh b/docs/pre-process.sh new file mode 100755 index 0000000000..75e9611f2f --- /dev/null +++ b/docs/pre-process.sh @@ -0,0 +1,61 @@ +#!/bin/bash -e + +# Populate an array with just docker dirs and one with content dirs +docker_dir=(`ls -d /docs/content/docker/*`) +content_dir=(`ls -d /docs/content/*`) + +# Loop content not of docker/ +# +# Sed to process GitHub Markdown +# 1-2 Remove comment code from metadata block +# 3 Remove .md extension from link text +# 4 Change ](/ to ](/project/ in links +# 5 Change ](word) to ](/project/word) +# 6 Change ](../../ to ](/project/ +# 7 Change ](../ to ](/project/word) +# +for i in "${content_dir[@]}" +do + : + case $i in + "/docs/content/windows") + ;; + "/docs/content/mac") + ;; + "/docs/content/linux") + ;; + "/docs/content/docker") + y=${i##*/} + find $i -type f -name "*.md" -exec sed -i.old \ + -e '/^/g' \ + -e '/^/g' {} \; + ;; + *) + y=${i##*/} + find $i -type f -name "*.md" -exec sed -i.old \ + -e '/^/g' \ + -e '/^/g' \ + -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/'$y'\//g' \ + -e 's/\(\][(]\)\([A-z].*\)\(\.md\)/\1\/'$y'\/\2/g' \ + -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ + -e 's/\(\][(]\)\(\.\/\)/\1\/'$y'\//g' \ + -e 's/\(\][(]\)\(\.\.\/\.\.\/\)/\1\/'$y'\//g' \ + -e 's/\(\][(]\)\(\.\.\/\)/\1\/'$y'\//g' {} \; + ;; + esac +done + +# +# Move docker directories to content +# +for i in "${docker_dir[@]}" +do + : + if [ -d $i ] + then + mv $i /docs/content/ + fi +done + +rm -rf /docs/content/docker + diff --git a/docs/reference/build.md b/docs/reference/build.md index b2b015119d..b6e27bb264 100644 --- a/docs/reference/build.md +++ b/docs/reference/build.md @@ -4,6 +4,7 @@ title = "build" description = "build" keywords = ["fig, composition, compose, docker, orchestration, cli, build"] [menu.main] +identifier="build.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/help.md b/docs/reference/help.md index 229ac5de94..613708ed2f 100644 --- a/docs/reference/help.md +++ b/docs/reference/help.md @@ -4,6 +4,7 @@ title = "help" description = "help" keywords = ["fig, composition, compose, docker, orchestration, cli, help"] [menu.main] +identifier="help.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/kill.md b/docs/reference/kill.md index c71608748c..e5dd057361 100644 --- a/docs/reference/kill.md +++ b/docs/reference/kill.md @@ -4,6 +4,7 @@ title = "kill" description = "Forces running containers to stop." keywords = ["fig, composition, compose, docker, orchestration, cli, kill"] [menu.main] +identifier="kill.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/logs.md b/docs/reference/logs.md index 87f937273f..5b241ea70b 100644 --- a/docs/reference/logs.md +++ b/docs/reference/logs.md @@ -4,6 +4,7 @@ title = "logs" description = "Displays log output from services." keywords = ["fig, composition, compose, docker, orchestration, cli, logs"] [menu.main] +identifier="logs.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/port.md b/docs/reference/port.md index 4745c92d32..76f93f2393 100644 --- a/docs/reference/port.md +++ b/docs/reference/port.md @@ -4,6 +4,7 @@ title = "port" description = "Prints the public port for a port binding.s" keywords = ["fig, composition, compose, docker, orchestration, cli, port"] [menu.main] +identifier="port.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/ps.md b/docs/reference/ps.md index b271376f80..546d68e76c 100644 --- a/docs/reference/ps.md +++ b/docs/reference/ps.md @@ -4,6 +4,7 @@ title = "ps" description = "Lists containers." keywords = ["fig, composition, compose, docker, orchestration, cli, ps"] [menu.main] +identifier="ps.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/pull.md b/docs/reference/pull.md index ac22010ec6..e5b5d166ff 100644 --- a/docs/reference/pull.md +++ b/docs/reference/pull.md @@ -4,6 +4,7 @@ title = "pull" description = "Pulls service images." keywords = ["fig, composition, compose, docker, orchestration, cli, pull"] [menu.main] +identifier="pull.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/restart.md b/docs/reference/restart.md index 9b570082bf..bbd4a68b0f 100644 --- a/docs/reference/restart.md +++ b/docs/reference/restart.md @@ -4,6 +4,7 @@ title = "restart" description = "Restarts Docker Compose services." keywords = ["fig, composition, compose, docker, orchestration, cli, restart"] [menu.main] +identifier="restart.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/rm.md b/docs/reference/rm.md index 0a4ba5b6bc..2ed959e411 100644 --- a/docs/reference/rm.md +++ b/docs/reference/rm.md @@ -4,6 +4,7 @@ title = "rm" description = "Removes stopped service containers." keywords = ["fig, composition, compose, docker, orchestration, cli, rm"] [menu.main] +identifier="rm.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/run.md b/docs/reference/run.md index b07ddd060d..5ea9a61bec 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -4,6 +4,7 @@ title = "run" description = "Runs a one-off command on a service." keywords = ["fig, composition, compose, docker, orchestration, cli, run"] [menu.main] +identifier="run.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/start.md b/docs/reference/start.md index 69d853f9cb..f0bdd5a97c 100644 --- a/docs/reference/start.md +++ b/docs/reference/start.md @@ -4,6 +4,7 @@ title = "start" description = "Starts existing containers for a service." keywords = ["fig, composition, compose, docker, orchestration, cli, start"] [menu.main] +identifier="start.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/stop.md b/docs/reference/stop.md index 8ff92129d2..ec7e6688a5 100644 --- a/docs/reference/stop.md +++ b/docs/reference/stop.md @@ -4,6 +4,7 @@ title = "stop" description = "Stops running containers without removing them. " keywords = ["fig, composition, compose, docker, orchestration, cli, stop"] [menu.main] +identifier="stop.compose" parent = "smn_compose_cli" +++ diff --git a/docs/reference/up.md b/docs/reference/up.md index 441d7f9c30..966aff1e95 100644 --- a/docs/reference/up.md +++ b/docs/reference/up.md @@ -4,6 +4,7 @@ title = "up" description = "Builds, (re)creates, starts, and attaches to containers for a service." keywords = ["fig, composition, compose, docker, orchestration, cli, up"] [menu.main] +identifier="up.compose" parent = "smn_compose_cli" +++ From 430ba8cda34107237d6cdca6fdc8ecbda9e0fbc6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 28 Jul 2015 15:06:22 +0100 Subject: [PATCH 004/337] Remove custom docs script Signed-off-by: Aanand Prasad --- script/docs | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100755 script/docs diff --git a/script/docs b/script/docs deleted file mode 100755 index 31c58861d0..0000000000 --- a/script/docs +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -ex - -# import the existing docs build cmds from docker/docker -DOCSPORT=8000 -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) -DOCKER_DOCS_IMAGE="compose-docs$GIT_BRANCH" -DOCKER_RUN_DOCS="docker run --rm -it -e NOCACHE" - -docker build -t "$DOCKER_DOCS_IMAGE" -f docs/Dockerfile . -$DOCKER_RUN_DOCS -p $DOCSPORT:8000 "$DOCKER_DOCS_IMAGE" mkdocs serve From b08e23d3519125f512cd9a6aff0f01e363d42ea1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 28 Jul 2015 15:31:18 +0100 Subject: [PATCH 005/337] Add hint about OS X binary compatibility Signed-off-by: Aanand Prasad --- docs/install.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/install.md b/docs/install.md index dad6efd56d..38302485a1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -35,6 +35,8 @@ To install Compose, run the following commands: Optionally, you can also install [command completion](completion.md) for the bash and zsh shell. +> **Note:** Some older Mac OS X CPU architectures are incompatible with the binary. If you receive an "Illegal instruction: 4" error after installing, you should install using the `pip` command instead. + Compose is available for OS X and 64-bit Linux. If you're on another platform, Compose can also be installed as a Python package: From fc203d643aa9a69c835aebee0de9b17851ef7a58 Mon Sep 17 00:00:00 2001 From: Reilly Herrewig-Pope Date: Tue, 28 Jul 2015 14:12:20 -0400 Subject: [PATCH 006/337] Allow API version specification via env var Hard-coding the API version to '1.18' with the docker-py constructor will cause the docker-py logic at https://github.com/docker/docker-py/blob/master/docker/client.py#L143-L146 to always fail, which will cause authentication issues if you're using a remote daemon using API version 1.19 - regardless of the API version of the registry. Allow the user to set the API version via an environment variable. If the variable is not present, it will still default to '1.18' like it does today. Signed-off-by: Reilly Herrewig-Pope --- compose/cli/docker_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index e513182fb3..adee9365b1 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -14,6 +14,8 @@ def docker_client(): cert_path = os.path.join(os.environ.get('HOME', ''), '.docker') base_url = os.environ.get('DOCKER_HOST') + api_version = os.environ.get('COMPOSE_API_VERSION', '1.18') + tls_config = None if os.environ.get('DOCKER_TLS_VERIFY', '') != '': @@ -32,4 +34,4 @@ def docker_client(): ) timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) - return Client(base_url=base_url, tls=tls_config, version='1.18', timeout=timeout) + return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=timeout) From 118a389646a68b914a2de0efb763d6d71868d951 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 28 Jul 2015 14:51:27 +0100 Subject: [PATCH 007/337] Update API version to 1.19 Signed-off-by: Aanand Prasad --- Dockerfile | 6 ++---- compose/cli/docker_client.py | 2 +- docs/install.md | 2 +- tests/integration/service_test.py | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 738e0b9978..a0e7f14f91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,16 +48,14 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.6.2 1.7.1 +ENV ALL_DOCKER_VERSIONS 1.7.1 RUN set -ex; \ - curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.2 -o /usr/local/bin/docker-1.6.2; \ - chmod +x /usr/local/bin/docker-1.6.2; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.7.1 -o /usr/local/bin/docker-1.7.1; \ chmod +x /usr/local/bin/docker-1.7.1 # Set the default Docker to be run -RUN ln -s /usr/local/bin/docker-1.6.2 /usr/local/bin/docker +RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index adee9365b1..244bcbef2f 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -14,7 +14,7 @@ def docker_client(): cert_path = os.path.join(os.environ.get('HOME', ''), '.docker') base_url = os.environ.get('DOCKER_HOST') - api_version = os.environ.get('COMPOSE_API_VERSION', '1.18') + api_version = os.environ.get('COMPOSE_API_VERSION', '1.19') tls_config = None diff --git a/docs/install.md b/docs/install.md index 38302485a1..adb32fd50b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -17,7 +17,7 @@ Compose with a `curl` command. ## Install Docker -First, install Docker version 1.6 or greater: +First, install Docker version 1.7.1 or greater: - [Instructions for Mac OS X](http://docs.docker.com/installation/mac/) - [Instructions for Ubuntu](http://docs.docker.com/installation/ubuntulinux/) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 60e2eed1f4..a901fc59ad 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -121,7 +121,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', cpu_shares=73) container = service.create_container() service.start_container(container) - self.assertEqual(container.inspect()['Config']['CpuShares'], 73) + self.assertEqual(container.get('HostConfig.CpuShares'), 73) def test_build_extra_hosts(self): # string @@ -183,7 +183,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', cpuset='0') container = service.create_container() service.start_container(container) - self.assertEqual(container.inspect()['Config']['Cpuset'], '0') + self.assertEqual(container.get('HostConfig.CpusetCpus'), '0') def test_create_container_with_read_only_root_fs(self): read_only = True From 976887250708fe1ad4f8c478cc5781c04655b92b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 29 Jul 2015 18:04:19 +0100 Subject: [PATCH 008/337] Fix "Duplicate volume mount" error when config has trailing slashes When an image declares a volume such as `/var/lib/mysql`, and a Compose file has a line like `./data:/var/lib/mysql/` (note the trailing slash), Compose creates duplicate volume binds when *recreating* the container. (The first container is created without a hitch, but contains multiple entries in its "Volumes" config.) Fixed by normalizing all paths in volumes config. Signed-off-by: Aanand Prasad --- compose/service.py | 12 +++++++---- tests/integration/service_test.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index b9b4ed3e0e..2e0490a508 100644 --- a/compose/service.py +++ b/compose/service.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from collections import namedtuple import logging import re +import os import sys from operator import attrgetter @@ -848,12 +849,15 @@ def parse_volume_spec(volume_config): "external:internal[:mode]" % volume_config) if len(parts) == 1: - return VolumeSpec(None, parts[0], 'rw') + external = None + internal = os.path.normpath(parts[0]) + else: + external = os.path.normpath(parts[0]) + internal = os.path.normpath(parts[1]) - if len(parts) == 2: - parts.append('rw') + mode = parts[2] if len(parts) == 3 else 'rw' - return VolumeSpec(*parts) + return VolumeSpec(external, internal, mode) # Ports diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a901fc59ad..b975fc00d0 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -221,6 +221,40 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + def test_duplicate_volume_trailing_slash(self): + """ + When an image specifies a volume, and the Compose file specifies a host path + but adds a trailing slash, make sure that we don't create duplicate binds. + """ + host_path = '/tmp/data' + container_path = '/data' + volumes = ['{}:{}/'.format(host_path, container_path)] + + tmp_container = self.client.create_container( + 'busybox', 'true', + volumes={container_path: {}}, + labels={'com.docker.compose.test_image': 'true'}, + ) + image = self.client.commit(tmp_container)['Id'] + + service = self.create_service('db', image=image, volumes=volumes) + old_container = create_and_start_container(service) + + self.assertEqual( + old_container.get('Config.Volumes'), + {container_path: {}}, + ) + + service = self.create_service('db', image=image, volumes=volumes) + new_container = service.recreate_container(old_container) + + self.assertEqual( + new_container.get('Config.Volumes'), + {container_path: {}}, + ) + + self.assertEqual(service.containers(stopped=False), [new_container]) + @patch.dict(os.environ) def test_create_container_with_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' From 03c3d4c768198bea7eedcde79c01177441e8a0c1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 22 Jul 2015 15:09:24 +0100 Subject: [PATCH 009/337] generator -> iterator Signed-off-by: Aanand Prasad --- compose/cli/multiplexer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 849dbd66a2..02e39aa1e6 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -13,8 +13,13 @@ STOP = object() class Multiplexer(object): - def __init__(self, generators): - self.generators = generators + """ + Create a single iterator from several iterators by running all of them in + parallel and yielding results as they come in. + """ + + def __init__(self, iterators): + self.iterators = iterators self.queue = Queue() def loop(self): @@ -31,12 +36,12 @@ class Multiplexer(object): pass def _init_readers(self): - for generator in self.generators: - t = Thread(target=_enqueue_output, args=(generator, self.queue)) + for iterator in self.iterators: + t = Thread(target=_enqueue_output, args=(iterator, self.queue)) t.daemon = True t.start() -def _enqueue_output(generator, queue): - for item in generator: +def _enqueue_output(iterator, queue): + for item in iterator: queue.put(item) From 27378704df946bd4f3bd994f750916c04e0dc139 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 22 Jul 2015 14:09:12 +0100 Subject: [PATCH 010/337] Isolate STOP logic in multiplexer module Signed-off-by: Aanand Prasad --- compose/cli/log_printer.py | 3 +-- compose/cli/multiplexer.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index ce7e106533..9c5d35e187 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,7 +4,7 @@ import sys from itertools import cycle -from .multiplexer import Multiplexer, STOP +from .multiplexer import Multiplexer from . import colors from .utils import split_buffer @@ -61,7 +61,6 @@ class LogPrinter(object): exit_code = container.wait() yield color_fn("%s exited with code %s\n" % (container.name, exit_code)) - yield STOP def _generate_prefix(self, container): """ diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 02e39aa1e6..ab7482e1dc 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -45,3 +45,5 @@ class Multiplexer(object): def _enqueue_output(iterator, queue): for item in iterator: queue.put(item) + + queue.put(STOP) From a9942b512a4fc6b04c334c821804a955a6c45ec0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 22 Jul 2015 14:08:46 +0100 Subject: [PATCH 011/337] Wait for all containers to exit when running 'up' interactively Signed-off-by: Aanand Prasad --- compose/cli/multiplexer.py | 7 +++---- tests/unit/multiplexer_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/unit/multiplexer_test.py diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index ab7482e1dc..34b55133cc 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -7,8 +7,6 @@ except ImportError: from queue import Queue, Empty # Python 3.x -# Yield STOP from an input generator to stop the -# top-level loop without processing any more input. STOP = object() @@ -20,16 +18,17 @@ class Multiplexer(object): def __init__(self, iterators): self.iterators = iterators + self._num_running = len(iterators) self.queue = Queue() def loop(self): self._init_readers() - while True: + while self._num_running > 0: try: item = self.queue.get(timeout=0.1) if item is STOP: - break + self._num_running -= 1 else: yield item except Empty: diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py new file mode 100644 index 0000000000..100b8f0c2d --- /dev/null +++ b/tests/unit/multiplexer_test.py @@ -0,0 +1,28 @@ +import unittest + +from compose.cli.multiplexer import Multiplexer + + +class MultiplexerTest(unittest.TestCase): + def test_no_iterators(self): + mux = Multiplexer([]) + self.assertEqual([], list(mux.loop())) + + def test_empty_iterators(self): + mux = Multiplexer([ + (x for x in []), + (x for x in []), + ]) + + self.assertEqual([], list(mux.loop())) + + def test_aggregates_output(self): + mux = Multiplexer([ + (x for x in [0, 2, 4]), + (x for x in [1, 3, 5]), + ]) + + self.assertEqual( + [0, 1, 2, 3, 4, 5], + sorted(list(mux.loop())), + ) From 80d90a745ad9816817a14f4aa35c3d9a1a2136b4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 22 Jul 2015 14:47:59 +0100 Subject: [PATCH 012/337] Make sure an exception in any iterator gets raised in the main thread Signed-off-by: Aanand Prasad Conflicts: compose/cli/multiplexer.py --- compose/cli/multiplexer.py | 16 +++++++++++----- tests/unit/multiplexer_test.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 34b55133cc..955af63221 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -26,7 +26,11 @@ class Multiplexer(object): while self._num_running > 0: try: - item = self.queue.get(timeout=0.1) + item, exception = self.queue.get(timeout=0.1) + + if exception: + raise exception + if item is STOP: self._num_running -= 1 else: @@ -42,7 +46,9 @@ class Multiplexer(object): def _enqueue_output(iterator, queue): - for item in iterator: - queue.put(item) - - queue.put(STOP) + try: + for item in iterator: + queue.put((item, None)) + queue.put((STOP, None)) + except Exception as e: + queue.put((None, e)) diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py index 100b8f0c2d..d565d39d1b 100644 --- a/tests/unit/multiplexer_test.py +++ b/tests/unit/multiplexer_test.py @@ -26,3 +26,20 @@ class MultiplexerTest(unittest.TestCase): [0, 1, 2, 3, 4, 5], sorted(list(mux.loop())), ) + + def test_exception(self): + class Problem(Exception): + pass + + def problematic_iterator(): + yield 0 + yield 2 + raise Problem(":(") + + mux = Multiplexer([ + problematic_iterator(), + (x for x in [1, 3, 5]), + ]) + + with self.assertRaises(Problem): + list(mux.loop()) From 27bd987f286209737c665dd355535e76d1e4e71e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 30 Jul 2015 10:31:54 +0100 Subject: [PATCH 013/337] Add test for trailing slash volume copying bug Signed-off-by: Aanand Prasad --- tests/integration/service_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index b975fc00d0..abab7c579d 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -221,6 +221,18 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + def test_recreate_preserves_volume_with_trailing_slash(self): + """ + When the Compose file specifies a trailing slash in the container path, make + sure we copy the volume over when recreating. + """ + service = self.create_service('data', volumes=['/data/']) + old_container = create_and_start_container(service) + volume_path = old_container.get('Volumes')['/data'] + + new_container = service.recreate_container(old_container) + self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + def test_duplicate_volume_trailing_slash(self): """ When an image specifies a volume, and the Compose file specifies a host path From 1a9ddf645d69cca77b88b4a0c6a38e5c2c841566 Mon Sep 17 00:00:00 2001 From: David BF Date: Fri, 31 Jul 2015 14:26:42 +0200 Subject: [PATCH 014/337] Remove useless postgres 'port' configuration Signed-off-by: David BF --- docs/rails.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/rails.md b/docs/rails.md index 7394aadc8a..9ce6c4a6f8 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -40,8 +40,6 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th db: image: postgres - ports: - - "5432" web: build: . command: bundle exec rails s -p 3000 -b '0.0.0.0' From a68ee0d9c2d3969e161ca973cfbe3e62bcb3dd2e Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 3 Jun 2015 12:21:29 +0100 Subject: [PATCH 015/337] Support volume_driver in compose * Add support for volume_driver parameter in compose yml * Don't expand volume host paths if a volume_driver is specified (i.e., disable compose feature "relative to absolute path transformation" when volume drivers are in use, since volume drivers can use name where host path is normally specified; this is a heuristic) Signed-off-by: Luke Marsden --- compose/config.py | 3 ++- docs/yml.md | 10 +++++++++- tests/integration/service_test.py | 6 ++++++ tests/unit/config_test.py | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/compose/config.py b/compose/config.py index 064dadaec4..af8983961f 100644 --- a/compose/config.py +++ b/compose/config.py @@ -43,6 +43,7 @@ DOCKER_CONFIG_KEYS = [ 'stdin_open', 'tty', 'user', + 'volume_driver', 'volumes', 'volumes_from', 'working_dir', @@ -251,7 +252,7 @@ def process_container_options(service_dict, working_dir=None): if 'memswap_limit' in service_dict and 'mem_limit' not in service_dict: raise ConfigurationError("Invalid 'memswap_limit' configuration for %s service: when defining 'memswap_limit' you must set 'mem_limit' as well" % service_dict['name']) - if 'volumes' in service_dict: + if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(service_dict['volumes'], working_dir=working_dir) if 'build' in service_dict: diff --git a/docs/yml.md b/docs/yml.md index f92b568256..f89d107bdc 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -134,6 +134,12 @@ Mount paths as volumes, optionally specifying a path on the host machine - cache/:/tmp/cache - ~/configs:/etc/configs/:ro +You can mount a relative path on the host, which will expand relative to +the directory of the Compose configuration file being used. + +> Note: No path expansion will be done if you have also specified a +> `volume_driver`. + ### volumes_from Mount all of the volumes from another service or container. @@ -333,7 +339,7 @@ Override the default labeling scheme for each container. - label:user:USER - label:role:ROLE -### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only +### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only, volume\_driver Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. @@ -360,6 +366,8 @@ Each of these is a single value, analogous to its tty: true read_only: true + volume_driver: mydriver +``` ## Compose documentation diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index abab7c579d..8856d0245f 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -117,6 +117,12 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertIn('/var/db', container.get('Volumes')) + def test_create_container_with_volume_driver(self): + service = self.create_service('db', volume_driver='foodriver') + container = service.create_container() + service.start_container(container) + self.assertEqual('foodriver', container.get('Config.VolumeDriver')) + def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) container = service.create_container() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 281717db74..a2c17d7254 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -72,6 +72,22 @@ class VolumePathTest(unittest.TestCase): d = make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') self.assertEqual(d['volumes'], ['/home/user:/container/path']) + def test_named_volume_with_driver(self): + d = make_service_dict('foo', { + 'volumes': ['namedvolume:/data'], + 'volume_driver': 'foodriver', + }, working_dir='.') + self.assertEqual(d['volumes'], ['namedvolume:/data']) + + @mock.patch.dict(os.environ) + def test_named_volume_with_special_chars(self): + os.environ['NAME'] = 'surprise!' + d = make_service_dict('foo', { + 'volumes': ['~/${NAME}:/data'], + 'volume_driver': 'foodriver', + }, working_dir='.') + self.assertEqual(d['volumes'], ['~/${NAME}:/data']) + class MergePathMappingTest(object): def config_name(self): From 92ef1f57022008d0bb5ed47971bccb83ed07afa4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 28 Jul 2015 16:48:38 +0100 Subject: [PATCH 016/337] Make compose.config a proper module Signed-off-by: Aanand Prasad --- compose/config/__init__.py | 10 ++++++++++ compose/{ => config}/config.py | 0 tests/integration/testcases.py | 2 +- tests/unit/config_test.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 compose/config/__init__.py rename compose/{ => config}/config.py (100%) diff --git a/compose/config/__init__.py b/compose/config/__init__.py new file mode 100644 index 0000000000..3907e5b67e --- /dev/null +++ b/compose/config/__init__.py @@ -0,0 +1,10 @@ +from .config import ( + DOCKER_CONFIG_KEYS, + ConfigDetails, + ConfigurationError, + find, + load, + parse_environment, + merge_environment, + get_service_name_from_net, +) # flake8: noqa diff --git a/compose/config.py b/compose/config/config.py similarity index 100% rename from compose/config.py rename to compose/config/config.py diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 2a7c0a440d..a7929088be 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from compose.service import Service -from compose.config import ServiceLoader +from compose.config.config import ServiceLoader from compose.const import LABEL_PROJECT from compose.cli.docker_client import docker_client from compose.progress_stream import stream_output diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index a2c17d7254..3ee754e319 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -4,7 +4,7 @@ import shutil import tempfile from .. import unittest -from compose import config +from compose.config import config def make_service_dict(name, service_dict, working_dir): From 31ac3ce22a381061b13046a4231161ed5c1a9eb3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 28 Jul 2015 18:05:39 +0100 Subject: [PATCH 017/337] Split out compose.config.errors Signed-off-by: Aanand Prasad --- compose/config/config.py | 36 ++++++------------------------------ compose/config/errors.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-) create mode 100644 compose/config/errors.py diff --git a/compose/config/config.py b/compose/config/config.py index af8983961f..d36967825c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -8,6 +8,12 @@ import six from compose.cli.utils import find_candidates_in_parent_dirs +from .errors import ( + ConfigurationError, + CircularReference, + ComposeFileNotFound, +) + DOCKER_CONFIG_KEYS = [ 'cap_add', @@ -536,33 +542,3 @@ def load_yaml(filename): return yaml.safe_load(fh) except IOError as e: raise ConfigurationError(six.text_type(e)) - - -class ConfigurationError(Exception): - def __init__(self, msg): - self.msg = msg - - def __str__(self): - return self.msg - - -class CircularReference(ConfigurationError): - def __init__(self, trail): - self.trail = trail - - @property - def msg(self): - lines = [ - "{} in {}".format(service_name, filename) - for (filename, service_name) in self.trail - ] - return "Circular reference:\n {}".format("\n extends ".join(lines)) - - -class ComposeFileNotFound(ConfigurationError): - def __init__(self, supported_filenames): - super(ComposeFileNotFound, self).__init__(""" - Can't find a suitable configuration file in this directory or any parent. Are you in the right directory? - - Supported filenames: %s - """ % ", ".join(supported_filenames)) diff --git a/compose/config/errors.py b/compose/config/errors.py new file mode 100644 index 0000000000..037b7ec84d --- /dev/null +++ b/compose/config/errors.py @@ -0,0 +1,28 @@ +class ConfigurationError(Exception): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + + +class CircularReference(ConfigurationError): + def __init__(self, trail): + self.trail = trail + + @property + def msg(self): + lines = [ + "{} in {}".format(service_name, filename) + for (filename, service_name) in self.trail + ] + return "Circular reference:\n {}".format("\n extends ".join(lines)) + + +class ComposeFileNotFound(ConfigurationError): + def __init__(self, supported_filenames): + super(ComposeFileNotFound, self).__init__(""" + Can't find a suitable configuration file in this directory or any parent. Are you in the right directory? + + Supported filenames: %s + """ % ", ".join(supported_filenames)) From 8b5bd945d0883ef71b87ca80e75c57e2636183a9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 24 Jul 2015 15:58:18 +0100 Subject: [PATCH 018/337] Interpolate environment variables Signed-off-by: Aanand Prasad --- compose/config/config.py | 9 ++- compose/config/interpolation.py | 69 +++++++++++++++++ docs/yml.md | 31 ++++++++ .../docker-compose.yml | 17 +++++ .../docker-compose.yml | 5 ++ tests/integration/cli_test.py | 15 ++++ tests/integration/service_test.py | 18 ----- tests/unit/config_test.py | 75 +++++++++++++++---- tests/unit/interpolation_test.py | 31 ++++++++ 9 files changed, 235 insertions(+), 35 deletions(-) create mode 100644 compose/config/interpolation.py create mode 100644 tests/fixtures/environment-interpolation/docker-compose.yml create mode 100644 tests/fixtures/volume-path-interpolation/docker-compose.yml create mode 100644 tests/unit/interpolation_test.py diff --git a/compose/config/config.py b/compose/config/config.py index d36967825c..4d3f5faefa 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -8,6 +8,7 @@ import six from compose.cli.utils import find_candidates_in_parent_dirs +from .interpolation import interpolate_environment_variables from .errors import ( ConfigurationError, CircularReference, @@ -132,11 +133,11 @@ def get_config_path(base_dir): def load(config_details): dictionary, working_dir, filename = config_details + dictionary = interpolate_environment_variables(dictionary) + service_dicts = [] for service_name, service_dict in list(dictionary.items()): - if not isinstance(service_dict, dict): - raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name) loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) validate_paths(service_dict) @@ -429,9 +430,9 @@ def resolve_volume_paths(volumes, working_dir=None): def resolve_volume_path(volume, working_dir): container_path, host_path = split_path_mapping(volume) - container_path = os.path.expanduser(os.path.expandvars(container_path)) + container_path = os.path.expanduser(container_path) if host_path is not None: - host_path = os.path.expanduser(os.path.expandvars(host_path)) + host_path = os.path.expanduser(host_path) return "%s:%s" % (expand_path(working_dir, host_path), container_path) else: return container_path diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py new file mode 100644 index 0000000000..0d4b96419c --- /dev/null +++ b/compose/config/interpolation.py @@ -0,0 +1,69 @@ +import os +from string import Template +from collections import defaultdict + +import six + +from .errors import ConfigurationError + + +def interpolate_environment_variables(config): + return dict( + (service_name, process_service(service_name, service_dict)) + for (service_name, service_dict) in config.items() + ) + + +def process_service(service_name, service_dict): + if not isinstance(service_dict, dict): + raise ConfigurationError( + 'Service "%s" doesn\'t have any configuration options. ' + 'All top level keys in your docker-compose.yml must map ' + 'to a dictionary of configuration options.' % service_name + ) + + return dict( + (key, interpolate_value(service_name, key, val)) + for (key, val) in service_dict.items() + ) + + +def interpolate_value(service_name, config_key, value): + try: + return recursive_interpolate(value) + except InvalidInterpolation as e: + raise ConfigurationError( + 'Invalid interpolation format for "{config_key}" option ' + 'in service "{service_name}": "{string}"' + .format( + config_key=config_key, + service_name=service_name, + string=e.string, + ) + ) + + +def recursive_interpolate(obj): + if isinstance(obj, six.string_types): + return interpolate(obj, os.environ) + elif isinstance(obj, dict): + return dict( + (key, recursive_interpolate(val)) + for (key, val) in obj.items() + ) + elif isinstance(obj, list): + return map(recursive_interpolate, obj) + else: + return obj + + +def interpolate(string, mapping): + try: + return Template(string).substitute(defaultdict(lambda: "", mapping)) + except ValueError: + raise InvalidInterpolation(string) + + +class InvalidInterpolation(Exception): + def __init__(self, string): + self.string = string diff --git a/docs/yml.md b/docs/yml.md index f89d107bdc..18551bf22f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -19,6 +19,10 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +Values for configuration options can contain environment variables, e.g. +`image: postgres:${POSTGRES_VERSION}`. For more details, see the section on +[variable substitution](#variable-substitution). + ### image Tag or partial image ID. Can be local or remote - Compose will attempt to @@ -369,6 +373,33 @@ Each of these is a single value, analogous to its volume_driver: mydriver ``` +## Variable substitution + +Your configuration options can contain environment variables. Compose uses the +variable values from the shell environment in which `docker-compose` is run. For +example, suppose the shell contains `POSTGRES_VERSION=9.3` and you supply this +configuration: + + db: + image: "postgres:${POSTGRES_VERSION}" + +When you run `docker-compose up` with this configuration, Compose looks for the +`POSTGRES_VERSION` environment variable in the shell and substitutes its value +in. For this example, Compose resolves the `image` to `postgres:9.3` before +running the configuration. + +If an environment variable is not set, Compose substitutes with an empty +string. In the example above, if `POSTGRES_VERSION` is not set, the value for +the `image` option is `postgres:`. + +Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style +features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not +supported. + +If you need to put a literal dollar sign in a configuration value, use a double +dollar sign (`$$`). + + ## Compose documentation - [User guide](/) diff --git a/tests/fixtures/environment-interpolation/docker-compose.yml b/tests/fixtures/environment-interpolation/docker-compose.yml new file mode 100644 index 0000000000..7ed43a812c --- /dev/null +++ b/tests/fixtures/environment-interpolation/docker-compose.yml @@ -0,0 +1,17 @@ +web: + # unbracketed name + image: $IMAGE + + # array element + ports: + - "${HOST_PORT}:8000" + + # dictionary item value + labels: + mylabel: "${LABEL_VALUE}" + + # unset value + hostname: "host-${UNSET_VALUE}" + + # escaped interpolation + command: "$${ESCAPED}" diff --git a/tests/fixtures/volume-path-interpolation/docker-compose.yml b/tests/fixtures/volume-path-interpolation/docker-compose.yml new file mode 100644 index 0000000000..6d4e236af9 --- /dev/null +++ b/tests/fixtures/volume-path-interpolation/docker-compose.yml @@ -0,0 +1,5 @@ +test: + image: busybox + command: top + volumes: + - "~/${VOLUME_NAME}:/container-path" diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index f3b3b9f5fb..0e86c2792f 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -488,6 +488,21 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) + @patch.dict(os.environ) + def test_home_and_env_var_in_volume_path(self): + os.environ['VOLUME_NAME'] = 'my-volume' + os.environ['HOME'] = '/tmp/home-dir' + expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME']) + + self.command.base_dir = 'tests/fixtures/volume-path-interpolation' + self.command.dispatch(['up', '-d'], None) + + container = self.project.containers(stopped=True)[0] + actual_host_path = container.get('Volumes')['/container-path'] + components = actual_host_path.split('/') + self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], + msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + def test_up_with_extends(self): self.command.base_dir = 'tests/fixtures/extends' self.command.dispatch(['up', '-d'], None) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8856d0245f..9bdc12f993 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -273,24 +273,6 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(service.containers(stopped=False), [new_container]) - @patch.dict(os.environ) - def test_create_container_with_home_and_env_var_in_volume_path(self): - os.environ['VOLUME_NAME'] = 'my-volume' - os.environ['HOME'] = '/tmp/home-dir' - expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME']) - - host_path = '~/${VOLUME_NAME}' - container_path = '/container-path' - - service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)]) - container = service.create_container() - service.start_container(container) - - actual_host_path = container.get('Volumes')[container_path] - components = actual_host_path.split('/') - self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], - msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) - def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') volume_container_1 = volume_service.create_container() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 3ee754e319..b1c22235b9 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -59,11 +59,56 @@ class ConfigTest(unittest.TestCase): make_service_dict('foo', {'ports': ['8000']}, 'tests/') -class VolumePathTest(unittest.TestCase): +class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) - def test_volume_binding_with_environ(self): + def test_config_file_with_environment_variable(self): + os.environ.update( + IMAGE="busybox", + HOST_PORT="80", + LABEL_VALUE="myvalue", + ) + + service_dicts = config.load( + config.find('tests/fixtures/environment-interpolation', None), + ) + + self.assertEqual(service_dicts, [ + { + 'name': 'web', + 'image': 'busybox', + 'ports': ['80:8000'], + 'labels': {'mylabel': 'myvalue'}, + 'hostname': 'host-', + 'command': '${ESCAPED}', + } + ]) + + @mock.patch.dict(os.environ) + def test_invalid_interpolation(self): + with self.assertRaises(config.ConfigurationError) as cm: + config.load( + config.ConfigDetails( + {'web': {'image': '${'}}, + 'working_dir', + 'filename.yml' + ) + ) + + self.assertIn('Invalid', cm.exception.msg) + self.assertIn('for "image" option', cm.exception.msg) + self.assertIn('in service "web"', cm.exception.msg) + self.assertIn('"${"', cm.exception.msg) + + @mock.patch.dict(os.environ) + def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' - d = make_service_dict('foo', {'volumes': ['${VOLUME_PATH}:/container/path']}, working_dir='.') + d = config.load( + config.ConfigDetails( + config={'foo': {'volumes': ['${VOLUME_PATH}:/container/path']}}, + working_dir='.', + filename=None, + ) + )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) @mock.patch.dict(os.environ) @@ -400,18 +445,22 @@ class EnvTest(unittest.TestCase): os.environ['HOSTENV'] = '/tmp' os.environ['CONTAINERENV'] = '/host/tmp' - service_dict = make_service_dict( - 'foo', - {'volumes': ['$HOSTENV:$CONTAINERENV']}, - working_dir="tests/fixtures/env" - ) + service_dict = config.load( + config.ConfigDetails( + config={'foo': {'volumes': ['$HOSTENV:$CONTAINERENV']}}, + working_dir="tests/fixtures/env", + filename=None, + ) + )[0] self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) - service_dict = make_service_dict( - 'foo', - {'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}, - working_dir="tests/fixtures/env" - ) + service_dict = config.load( + config.ConfigDetails( + config={'foo': {'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, + working_dir="tests/fixtures/env", + filename=None, + ) + )[0] self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py new file mode 100644 index 0000000000..96c6f9b33a --- /dev/null +++ b/tests/unit/interpolation_test.py @@ -0,0 +1,31 @@ +import unittest + +from compose.config.interpolation import interpolate, InvalidInterpolation + + +class InterpolationTest(unittest.TestCase): + def test_valid_interpolations(self): + self.assertEqual(interpolate('$foo', dict(foo='hi')), 'hi') + self.assertEqual(interpolate('${foo}', dict(foo='hi')), 'hi') + + self.assertEqual(interpolate('${subject} love you', dict(subject='i')), 'i love you') + self.assertEqual(interpolate('i ${verb} you', dict(verb='love')), 'i love you') + self.assertEqual(interpolate('i love ${object}', dict(object='you')), 'i love you') + + def test_empty_value(self): + self.assertEqual(interpolate('${foo}', dict(foo='')), '') + + def test_unset_value(self): + self.assertEqual(interpolate('${foo}', dict()), '') + + def test_escaped_interpolation(self): + self.assertEqual(interpolate('$${foo}', dict(foo='hi')), '${foo}') + + def test_invalid_strings(self): + self.assertRaises(InvalidInterpolation, lambda: interpolate('${', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('$}', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${}', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${ }', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${ foo}', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo }', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo!}', dict())) From ee6ff294a273d07e157af68f0b5f97f36b957676 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 6 Aug 2015 11:31:42 +0100 Subject: [PATCH 019/337] Show a warning when a variable is unset Signed-off-by: Aanand Prasad --- compose/config/interpolation.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 0d4b96419c..d33e93be49 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -1,11 +1,13 @@ import os from string import Template -from collections import defaultdict import six from .errors import ConfigurationError +import logging +log = logging.getLogger(__name__) + def interpolate_environment_variables(config): return dict( @@ -59,11 +61,26 @@ def recursive_interpolate(obj): def interpolate(string, mapping): try: - return Template(string).substitute(defaultdict(lambda: "", mapping)) + return Template(string).substitute(BlankDefaultDict(mapping)) except ValueError: raise InvalidInterpolation(string) +class BlankDefaultDict(dict): + def __init__(self, mapping): + super(BlankDefaultDict, self).__init__(mapping) + + def __getitem__(self, key): + try: + return super(BlankDefaultDict, self).__getitem__(key) + except KeyError: + log.warn( + "The {} variable is not set. Substituting a blank string." + .format(key) + ) + return "" + + class InvalidInterpolation(Exception): def __init__(self, string): self.string = string From 4f1429869462f61fd307ec63552b290e50b53882 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 6 Aug 2015 14:45:35 +0100 Subject: [PATCH 020/337] Abort tests if daemon fails to start Signed-off-by: Aanand Prasad --- script/wrapdocker | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/script/wrapdocker b/script/wrapdocker index 2e07bdadfd..119e88df4a 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -8,9 +8,16 @@ fi # delete it so that docker can start. rm -rf /var/run/docker.pid docker -d $DOCKER_DAEMON_ARGS &>/var/log/docker.log & +docker_pid=$! >&2 echo "Waiting for Docker to start..." while ! docker ps &>/dev/null; do + if ! kill -0 "$docker_pid" &>/dev/null; then + >&2 echo "Docker failed to start" + cat /var/log/docker.log + exit 1 + fi + sleep 1 done From fdaa5f2cde7e0721f26ca4e95cbb8f53402be4a7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 6 Aug 2015 16:14:37 +0100 Subject: [PATCH 021/337] Update volume tests for clarity - Better method names. - Environment variable syntax in volume paths, even when a driver is specified, now *will* be processed (the test wasn't testing it properly). However, `~` will still *not* expand to the user's home directory. Signed-off-by: Aanand Prasad --- tests/unit/config_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index b1c22235b9..0046202030 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -117,7 +117,7 @@ class InterpolationTest(unittest.TestCase): d = make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') self.assertEqual(d['volumes'], ['/home/user:/container/path']) - def test_named_volume_with_driver(self): + def test_named_volume_with_driver_does_not_expand(self): d = make_service_dict('foo', { 'volumes': ['namedvolume:/data'], 'volume_driver': 'foodriver', @@ -125,13 +125,13 @@ class InterpolationTest(unittest.TestCase): self.assertEqual(d['volumes'], ['namedvolume:/data']) @mock.patch.dict(os.environ) - def test_named_volume_with_special_chars(self): + def test_home_directory_with_driver_does_not_expand(self): os.environ['NAME'] = 'surprise!' d = make_service_dict('foo', { - 'volumes': ['~/${NAME}:/data'], + 'volumes': ['~:/data'], 'volume_driver': 'foodriver', }, working_dir='.') - self.assertEqual(d['volumes'], ['~/${NAME}:/data']) + self.assertEqual(d['volumes'], ['~:/data']) class MergePathMappingTest(object): From da36ee7cbcaf2051fc0829f273c01517bd7d9bc2 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 27 Jul 2015 15:15:07 +0100 Subject: [PATCH 022/337] Perform schema validation Define a schema that we can pass to jsonschema to validate against the config a user has supplied. This will help catch a wide variety of common errors that occur. If the config does not pass schema validation then it raises an exception and prints out human readable reasons. Signed-off-by: Mazz Mosley --- compose/config/config.py | 43 ++++--- compose/schema.json | 79 ++++++++++++ compose/service.py | 6 - requirements.txt | 1 + setup.py | 1 + .../fixtures/extends/specify-file-as-self.yml | 1 + tests/unit/config_test.py | 118 +++++++++++------- tests/unit/service_test.py | 1 - 8 files changed, 175 insertions(+), 75 deletions(-) create mode 100644 compose/schema.json diff --git a/compose/config/config.py b/compose/config/config.py index 4d3f5faefa..1e793d9f64 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -3,6 +3,8 @@ import os import sys import yaml from collections import namedtuple +import json +import jsonschema import six @@ -131,13 +133,31 @@ def get_config_path(base_dir): return os.path.join(path, winner) +def validate_against_schema(config): + config_source_dir = os.path.dirname(os.path.abspath(__file__)) + schema_file = os.path.join(config_source_dir, "schema.json") + + with open(schema_file, "r") as schema_fh: + schema = json.load(schema_fh) + + validation_output = jsonschema.Draft4Validator(schema) + + errors = [error.message for error in sorted(validation_output.iter_errors(config), key=str)] + if errors: + raise ConfigurationError("Validation failed, reason(s): {}".format("\n".join(errors))) + + def load(config_details): - dictionary, working_dir, filename = config_details - dictionary = interpolate_environment_variables(dictionary) + config, working_dir, filename = config_details + config = interpolate_environment_variables(config) service_dicts = [] - for service_name, service_dict in list(dictionary.items()): + validate_against_schema(config) + + for service_name, service_dict in list(config.items()): + if not isinstance(service_dict, dict): + raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name) loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) validate_paths(service_dict) @@ -210,25 +230,11 @@ class ServiceLoader(object): def validate_extends_options(self, service_name, extends_options): error_prefix = "Invalid 'extends' configuration for %s:" % service_name - if not isinstance(extends_options, dict): - raise ConfigurationError("%s must be a dictionary" % error_prefix) - - if 'service' not in extends_options: - raise ConfigurationError( - "%s you need to specify a service, e.g. 'service: web'" % error_prefix - ) - if 'file' not in extends_options and self.filename is None: raise ConfigurationError( "%s you need to specify a 'file', e.g. 'file: something.yml'" % error_prefix ) - for k, _ in extends_options.items(): - if k not in ['file', 'service']: - raise ConfigurationError( - "%s unsupported configuration option '%s'" % (error_prefix, k) - ) - return extends_options @@ -256,9 +262,6 @@ def process_container_options(service_dict, working_dir=None): service_dict = service_dict.copy() - if 'memswap_limit' in service_dict and 'mem_limit' not in service_dict: - raise ConfigurationError("Invalid 'memswap_limit' configuration for %s service: when defining 'memswap_limit' you must set 'mem_limit' as well" % service_dict['name']) - if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(service_dict['volumes'], working_dir=working_dir) diff --git a/compose/schema.json b/compose/schema.json new file mode 100644 index 0000000000..7c7e2d096c --- /dev/null +++ b/compose/schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "type": "object", + + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + + "definitions": { + "service": { + "type": "object", + + "properties": { + "build": {"type": "string"}, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": { + "oneOf": [ + {"type": "object"}, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + "image": {"type": "string"}, + "mem_limit": {"type": "number"}, + "memswap_limit": {"type": "number"}, + + "extends": { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + + }, + + "anyOf": [ + { + "required": ["build"], + "not": {"required": ["image"]} + }, + { + "required": ["image"], + "not": {"required": ["build"]} + }, + { + "required": ["extends"], + "not": {"required": ["build", "image"]} + } + ], + + "dependencies": { + "memswap_limit": ["mem_limit"] + } + + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + } + + }, + + "additionalProperties": false +} diff --git a/compose/service.py b/compose/service.py index 2e0490a508..c72365cf99 100644 --- a/compose/service.py +++ b/compose/service.py @@ -82,14 +82,8 @@ ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') class Service(object): def __init__(self, name, client=None, project='default', links=None, external_links=None, volumes_from=None, net=None, **options): - if not re.match('^%s+$' % VALID_NAME_CHARS, name): - raise ConfigError('Invalid service name "%s" - only %s are allowed' % (name, VALID_NAME_CHARS)) if not re.match('^%s+$' % VALID_NAME_CHARS, project): raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS)) - if 'image' in options and 'build' in options: - raise ConfigError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name) - if 'image' not in options and 'build' not in options: - raise ConfigError('Service %s has neither an image nor a build path specified. Exactly one must be provided.' % name) self.name = name self.client = client diff --git a/requirements.txt b/requirements.txt index f9cec8372c..6416876861 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML==3.10 +jsonschema==2.5.1 docker-py==1.3.1 dockerpty==0.3.4 docopt==0.6.1 diff --git a/setup.py b/setup.py index 9bca4752de..1f9c981d1b 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ 'docker-py >= 1.3.1, < 1.4', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', + 'jsonschema >= 2.5.1, < 3', ] diff --git a/tests/fixtures/extends/specify-file-as-self.yml b/tests/fixtures/extends/specify-file-as-self.yml index 7e24997623..c24f10bc92 100644 --- a/tests/fixtures/extends/specify-file-as-self.yml +++ b/tests/fixtures/extends/specify-file-as-self.yml @@ -12,5 +12,6 @@ web: environment: - "BAZ=3" otherweb: + image: busybox environment: - "YEP=1" diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0046202030..3ed394a92a 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -20,10 +20,10 @@ class ConfigTest(unittest.TestCase): config.ConfigDetails( { 'foo': {'image': 'busybox'}, - 'bar': {'environment': ['FOO=1']}, + 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, }, - 'working_dir', - 'filename.yml' + 'tests/fixtures/extends', + 'common.yml' ) ) @@ -32,13 +32,14 @@ class ConfigTest(unittest.TestCase): sorted([ { 'name': 'bar', + 'image': 'busybox', 'environment': {'FOO': '1'}, }, { 'name': 'foo', 'image': 'busybox', } - ]) + ], key=lambda d: d['name']) ) def test_load_throws_error_when_not_dict(self): @@ -327,23 +328,26 @@ class MemoryOptionsTest(unittest.TestCase): When you set a 'memswap_limit' it is invalid config unless you also set a mem_limit """ - with self.assertRaises(config.ConfigurationError): - make_service_dict( - 'foo', { - 'memswap_limit': 2000000, - }, - 'tests/' + with self.assertRaisesRegexp(config.ConfigurationError, "u'mem_limit' is a dependency of u'memswap_limit'"): + config.load( + config.ConfigDetails( + { + 'foo': {'image': 'busybox', 'memswap_limit': 2000000}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) ) def test_validation_with_correct_memswap_values(self): - service_dict = make_service_dict( - 'foo', { - 'mem_limit': 1000000, - 'memswap_limit': 2000000, - }, - 'tests/' + service_dict = config.load( + config.ConfigDetails( + {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}}, + 'tests/fixtures/extends', + 'common.yml' + ) ) - self.assertEqual(service_dict['memswap_limit'], 2000000) + self.assertEqual(service_dict[0]['memswap_limit'], 2000000) class EnvTest(unittest.TestCase): @@ -528,6 +532,7 @@ class ExtendsTest(unittest.TestCase): { 'environment': {'YEP': '1'}, + 'image': 'busybox', 'name': 'otherweb' }, { @@ -553,36 +558,47 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_empty_dictionary(self): - dictionary = {'extends': None} - - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - - self.assertRaisesRegexp(config.ConfigurationError, 'dictionary', load_config) - - dictionary['extends'] = {} - self.assertRaises(config.ConfigurationError, load_config) + with self.assertRaisesRegexp(config.ConfigurationError, 'service'): + config.load( + config.ConfigDetails( + { + 'web': {'image': 'busybox', 'extends': {}}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) def test_extends_validation_missing_service_key(self): - dictionary = {'extends': {'file': 'common.yml'}} - - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - - self.assertRaisesRegexp(config.ConfigurationError, 'service', load_config) + with self.assertRaisesRegexp(config.ConfigurationError, "u'service' is a required property"): + config.load( + config.ConfigDetails( + { + 'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) def test_extends_validation_invalid_key(self): - dictionary = { - 'extends': - { - 'service': 'web', 'file': 'common.yml', 'what': 'is this' - } - } - - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - - self.assertRaisesRegexp(config.ConfigurationError, 'what', load_config) + with self.assertRaisesRegexp(config.ConfigurationError, "'rogue_key' was unexpected"): + config.load( + config.ConfigDetails( + { + 'web': { + 'image': 'busybox', + 'extends': { + 'file': 'common.yml', + 'service': 'web', + 'rogue_key': 'is not allowed' + } + }, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) def test_extends_validation_no_file_key_no_filename_set(self): dictionary = {'extends': {'service': 'web'}} @@ -593,12 +609,18 @@ class ExtendsTest(unittest.TestCase): self.assertRaisesRegexp(config.ConfigurationError, 'file', load_config) def test_extends_validation_valid_config(self): - dictionary = {'extends': {'service': 'web', 'file': 'common.yml'}} + service = config.load( + config.ConfigDetails( + { + 'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}}, + }, + 'tests/fixtures/extends', + 'common.yml' + ) + ) - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - - self.assertIsInstance(load_config(), dict) + self.assertEquals(len(service), 1) + self.assertIsInstance(service[0], dict) def test_extends_file_defaults_to_self(self): """ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index bc6b9e485e..aa348466a4 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -49,7 +49,6 @@ class ServiceTest(unittest.TestCase): Service('.__.', image='foo') def test_project_validation(self): - self.assertRaises(ConfigError, lambda: Service('bar')) self.assertRaises(ConfigError, lambda: Service(name='foo', project='>', image='foo')) Service(name='foo', project='bar.bar__', image='foo') From 76e6029f2132cfd531d069e57bb2e33060e84eb5 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 29 Jul 2015 16:09:33 +0100 Subject: [PATCH 023/337] Replace service tests with config tests We validate the config against our schema before a service is created so checking whether a service name is valid at time of instantiation of the Service class is not needed. Signed-off-by: Mazz Mosley --- tests/unit/config_test.py | 21 +++++++++++++++++++++ tests/unit/service_test.py | 19 ------------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 3ed394a92a..f06cbab637 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -59,6 +59,27 @@ class ConfigTest(unittest.TestCase): ) make_service_dict('foo', {'ports': ['8000']}, 'tests/') + def test_config_invalid_service_names(self): + with self.assertRaises(config.ConfigurationError): + for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + config.load( + config.ConfigDetails( + {invalid_name: {'image': 'busybox'}}, + 'working_dir', + 'filename.yml' + ) + ) + + def test_config_valid_service_names(self): + for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: + config.load( + config.ConfigDetails( + {valid_name: {'image': 'busybox'}}, + 'tests/fixtures/extends', + 'common.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index aa348466a4..a99197e63c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -29,25 +29,6 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.Client) - def test_name_validations(self): - self.assertRaises(ConfigError, lambda: Service(name='', image='foo')) - - self.assertRaises(ConfigError, lambda: Service(name=' ', image='foo')) - self.assertRaises(ConfigError, lambda: Service(name='/', image='foo')) - self.assertRaises(ConfigError, lambda: Service(name='!', image='foo')) - self.assertRaises(ConfigError, lambda: Service(name='\xe2', image='foo')) - - Service('a', image='foo') - Service('foo', image='foo') - Service('foo-bar', image='foo') - Service('foo.bar', image='foo') - Service('foo_bar', image='foo') - Service('_', image='foo') - Service('___', image='foo') - Service('-', image='foo') - Service('--', image='foo') - Service('.__.', image='foo') - def test_project_validation(self): self.assertRaises(ConfigError, lambda: Service(name='foo', project='>', image='foo')) From 6c7c5985465d63a70e579ed3e253ca8d0f5d4b06 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 29 Jul 2015 16:37:11 +0100 Subject: [PATCH 024/337] Format validation of ports Signed-off-by: Mazz Mosley --- compose/config/config.py | 26 ++++++++++++++++++++++++-- compose/schema.json | 11 +++++++++++ tests/unit/config_test.py | 22 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 1e793d9f64..6cffa2fe88 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -4,7 +4,7 @@ import sys import yaml from collections import namedtuple import json -import jsonschema +from jsonschema import Draft4Validator, FormatChecker, ValidationError import six @@ -133,6 +133,28 @@ def get_config_path(base_dir): return os.path.join(path, winner) +@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted.")) +def format_ports(instance): + def _is_valid(port): + if ':' in port or '/' in port: + return True + try: + int(port) + return True + except ValueError: + return False + return False + + if isinstance(instance, list): + for port in instance: + if not _is_valid(port): + return False + return True + elif isinstance(instance, str): + return _is_valid(instance) + return False + + def validate_against_schema(config): config_source_dir = os.path.dirname(os.path.abspath(__file__)) schema_file = os.path.join(config_source_dir, "schema.json") @@ -140,7 +162,7 @@ def validate_against_schema(config): with open(schema_file, "r") as schema_fh: schema = json.load(schema_fh) - validation_output = jsonschema.Draft4Validator(schema) + validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"])) errors = [error.message for error in sorted(validation_output.iter_errors(config), key=str)] if errors: diff --git a/compose/schema.json b/compose/schema.json index 7c7e2d096c..bf43ca36b0 100644 --- a/compose/schema.json +++ b/compose/schema.json @@ -14,6 +14,17 @@ "type": "object", "properties": { + "ports": { + "oneOf": [ + {"type": "string", "format": "ports"}, + { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true, + "format": "ports" + } + ] + }, "build": {"type": "string"}, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": { diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f06cbab637..f7e949d3cb 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -80,6 +80,28 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_invalid_ports_format_validation(self): + with self.assertRaises(config.ConfigurationError): + for invalid_ports in [{"1": "8000"}, "whatport"]: + config.load( + config.ConfigDetails( + {'web': {'image': 'busybox', 'ports': invalid_ports}}, + 'working_dir', + 'filename.yml' + ) + ) + + def test_config_valid_ports_format_validation(self): + valid_ports = [["8000", "9000"], "625", "8000:8050", ["8000/8050"]] + for ports in valid_ports: + config.load( + config.ConfigDetails( + {'web': {'image': 'busybox', 'ports': ports}}, + 'working_dir', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 98c7a7da6110e72540810e888eec9d42c8172f9d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 30 Jul 2015 10:53:41 +0100 Subject: [PATCH 025/337] Order properties alphabetically Improves readability. Signed-off-by: Mazz Mosley --- compose/schema.json | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/compose/schema.json b/compose/schema.json index bf43ca36b0..3e719fc424 100644 --- a/compose/schema.json +++ b/compose/schema.json @@ -14,28 +14,15 @@ "type": "object", "properties": { - "ports": { - "oneOf": [ - {"type": "string", "format": "ports"}, - { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true, - "format": "ports" - } - ] - }, "build": {"type": "string"}, "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": { "oneOf": [ {"type": "object"}, {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] }, - "image": {"type": "string"}, - "mem_limit": {"type": "number"}, - "memswap_limit": {"type": "number"}, "extends": { "type": "object", @@ -46,6 +33,22 @@ }, "required": ["service"], "additionalProperties": false + }, + + "image": {"type": "string"}, + "mem_limit": {"type": "number"}, + "memswap_limit": {"type": "number"}, + + "ports": { + "oneOf": [ + {"type": "string", "format": "ports"}, + { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true, + "format": "ports" + } + ] } }, From 8d6694085d8e9a80223b6473e1f9a1939b3ef936 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 30 Jul 2015 17:11:28 +0100 Subject: [PATCH 026/337] Include remaining valid config properties Signed-off-by: Mazz Mosley --- compose/config/config.py | 5 ---- compose/schema.json | 59 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 6cffa2fe88..27f845b75b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -525,11 +525,6 @@ def parse_labels(labels): if isinstance(labels, dict): return labels - raise ConfigurationError( - "labels \"%s\" must be a list or mapping" % - labels - ) - def split_label(label): if '=' in label: diff --git a/compose/schema.json b/compose/schema.json index 3e719fc424..258f44ccac 100644 --- a/compose/schema.json +++ b/compose/schema.json @@ -15,6 +15,19 @@ "properties": { "build": {"type": "string"}, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "command": {"$ref": "#/definitions/string_or_list"}, + "container_name": {"type": "string"}, + "cpu_shares": {"type": "string"}, + "cpuset": {"type": "string"}, + "detach": {"type": "boolean"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "dockerfile": {"type": "string"}, + "domainname": {"type": "string"}, + "entrypoint": {"type": "string"}, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": { @@ -24,6 +37,8 @@ ] }, + "expose": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extends": { "type": "object", @@ -35,9 +50,29 @@ "additionalProperties": false }, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "hostname": {"type": "string"}, "image": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "log_driver": {"type": "string"}, + + "log_opt": { + "type": "object", + + "properties": { + "address": {"type": "string"} + }, + "required": ["address"] + }, + + "mac_address": {"type": "string"}, "mem_limit": {"type": "number"}, "memswap_limit": {"type": "number"}, + "name": {"type": "string"}, + "net": {"type": "string"}, + "pid": {"type": "string"}, "ports": { "oneOf": [ @@ -49,8 +84,18 @@ "format": "ports" } ] - } + }, + "privileged": {"type": "string"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "string"}, + "stdin_open": {"type": "string"}, + "tty": {"type": "string"}, + "user": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} }, "anyOf": [ @@ -70,8 +115,8 @@ "dependencies": { "memswap_limit": ["mem_limit"] - } - + }, + "additionalProperties": false }, "string_or_list": { @@ -85,9 +130,15 @@ "type": "array", "items": {"type": "string"}, "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + {"type": "object"} + ] } }, - "additionalProperties": false } From d8aee782c876e1e6aa1d31ebaaf4fe566018fc26 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 4 Aug 2015 17:43:33 +0100 Subject: [PATCH 027/337] Error handling jsonschema provides a rich error tree of info, by parsing each error we can pull out relevant info and re-write the error messages. This covers current error handling behaviour. This includes new error handling behaviour for types and formatting of the ports field. Signed-off-by: Mazz Mosley --- compose/config/config.py | 78 ++++++++++++++++++++++++++++++++++++++- compose/service.py | 4 +- tests/unit/config_test.py | 6 ++- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 27f845b75b..f2a89699d9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -18,6 +18,8 @@ from .errors import ( ) +VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' + DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', @@ -155,6 +157,77 @@ def format_ports(instance): return False +def get_unsupported_config_msg(service_name, error_key): + msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) + if error_key in DOCKER_CONFIG_HINTS: + msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) + return msg + + +def process_errors(errors): + """ + jsonschema gives us an error tree full of information to explain what has + gone wrong. Process each error and pull out relevant information and re-write + helpful error messages that are relevant. + """ + def _parse_key_from_error_msg(error): + return error.message.split("'")[1] + + root_msgs = [] + invalid_keys = [] + required = [] + type_errors = [] + + for error in errors: + # handle root level errors + if len(error.path) == 0: + if error.validator == 'type': + msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." + root_msgs.append(msg) + elif error.validator == 'additionalProperties': + invalid_service_name = _parse_key_from_error_msg(error) + msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS) + root_msgs.append(msg) + else: + root_msgs.append(error.message) + + else: + # handle service level errors + service_name = error.path[0] + + if error.validator == 'additionalProperties': + invalid_config_key = _parse_key_from_error_msg(error) + invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) + elif error.validator == 'anyOf': + if 'image' in error.instance and 'build' in error.instance: + required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name)) + elif 'image' not in error.instance and 'build' not in error.instance: + required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) + else: + required.append(error.message) + elif error.validator == 'type': + msg = "a" + if error.validator_value == "array": + msg = "an" + + try: + config_key = error.path[1] + type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value)) + except IndexError: + config_key = error.path[0] + root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key)) + elif error.validator == 'required': + config_key = error.path[1] + required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message)) + elif error.validator == 'dependencies': + dependency_key = error.validator_value.keys()[0] + required_keys = ",".join(error.validator_value[dependency_key]) + required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( + dependency_key, service_name, dependency_key, required_keys)) + + return "\n".join(root_msgs + invalid_keys + required + type_errors) + + def validate_against_schema(config): config_source_dir = os.path.dirname(os.path.abspath(__file__)) schema_file = os.path.join(config_source_dir, "schema.json") @@ -164,9 +237,10 @@ def validate_against_schema(config): validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"])) - errors = [error.message for error in sorted(validation_output.iter_errors(config), key=str)] + errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] if errors: - raise ConfigurationError("Validation failed, reason(s): {}".format("\n".join(errors))) + error_msg = process_errors(errors) + raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg)) def load(config_details): diff --git a/compose/service.py b/compose/service.py index c72365cf99..103840c3be 100644 --- a/compose/service.py +++ b/compose/service.py @@ -12,7 +12,7 @@ from docker.errors import APIError from docker.utils import create_host_config, LogConfig from . import __version__ -from .config import DOCKER_CONFIG_KEYS, merge_environment +from .config import DOCKER_CONFIG_KEYS, merge_environment, VALID_NAME_CHARS from .const import ( DEFAULT_TIMEOUT, LABEL_CONTAINER_NUMBER, @@ -49,8 +49,6 @@ DOCKER_START_KEYS = [ 'security_opt', ] -VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' - class BuildError(Exception): def __init__(self, service, reason): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f7e949d3cb..c0ccead8ab 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -371,7 +371,8 @@ class MemoryOptionsTest(unittest.TestCase): When you set a 'memswap_limit' it is invalid config unless you also set a mem_limit """ - with self.assertRaisesRegexp(config.ConfigurationError, "u'mem_limit' is a dependency of u'memswap_limit'"): + expected_error_msg = "Invalid 'memswap_limit' configuration for 'foo' service: when defining 'memswap_limit' you must set 'mem_limit' as well" + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -625,7 +626,8 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_invalid_key(self): - with self.assertRaisesRegexp(config.ConfigurationError, "'rogue_key' was unexpected"): + expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { From ea3608e1f4c5894ebbdc21fddeab4746deda05d8 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 5 Aug 2015 12:33:28 +0100 Subject: [PATCH 028/337] Improve test coverage for validation Signed-off-by: Mazz Mosley --- tests/unit/config_test.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index c0ccead8ab..15657f878f 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -102,6 +102,56 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_hint(self): + expected_error_msg = "(did you mean 'privileged'?)" + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'foo': {'image': 'busybox', 'privilige': 'something'}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + def test_invalid_config_build_and_image_specified(self): + expected_error_msg = "Service 'foo' has both an image and build path specified." + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'foo': {'image': 'busybox', 'build': '.'}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + def test_invalid_config_type_should_be_an_array(self): + expected_error_msg = "Service 'foo' has an invalid value for 'links', it should be an array" + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'foo': {'image': 'busybox', 'links': 'an_link'}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + def test_invalid_config_not_a_dictionary(self): + expected_error_msg = "Top level object needs to be a dictionary." + with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + ['foo', 'lol'], + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 0557b5dce6cbebe7bc24f415f4138d487524319b Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 5 Aug 2015 15:28:05 +0100 Subject: [PATCH 029/337] Remove dead code These functions weren't being called by anything. Signed-off-by: Mazz Mosley --- compose/config/config.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index f2a89699d9..31e5e9166c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -252,8 +252,6 @@ def load(config_details): validate_against_schema(config) for service_name, service_dict in list(config.items()): - if not isinstance(service_dict, dict): - raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name) loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) validate_paths(service_dict) @@ -427,18 +425,6 @@ def merge_environment(base, override): return env -def parse_links(links): - return dict(parse_link(l) for l in links) - - -def parse_link(link): - if ':' in link: - source, alias = link.split(':', 1) - return (alias, source) - else: - return (link, link) - - def get_env_files(options, working_dir=None): if 'env_file' not in options: return {} From 2e428f94ca3e0333a5b8b6469cb6fd528041cbe7 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 6 Aug 2015 16:56:45 +0100 Subject: [PATCH 030/337] Refactor validation out Move validation out into its own file without causing circular import errors. Fix some of the tests to import from the right place. Also fix tests that were not using valid test data, as the validation schema is now firing telling you that you couldn't "just" have this dict without a build/image config key. Signed-off-by: Mazz Mosley --- compose/config/config.py | 145 ++----------------------------- compose/{ => config}/schema.json | 0 compose/config/validation.py | 134 ++++++++++++++++++++++++++++ compose/service.py | 3 +- tests/unit/config_test.py | 50 +++++------ 5 files changed, 165 insertions(+), 167 deletions(-) rename compose/{ => config}/schema.json (100%) create mode 100644 compose/config/validation.py diff --git a/compose/config/config.py b/compose/config/config.py index 31e5e9166c..c1cfdb73d6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -3,9 +3,6 @@ import os import sys import yaml from collections import namedtuple -import json -from jsonschema import Draft4Validator, FormatChecker, ValidationError - import six from compose.cli.utils import find_candidates_in_parent_dirs @@ -16,10 +13,9 @@ from .errors import ( CircularReference, ComposeFileNotFound, ) +from .validation import validate_against_schema -VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' - DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', @@ -69,22 +65,6 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'name', ] -DOCKER_CONFIG_HINTS = { - 'cpu_share': 'cpu_shares', - 'add_host': 'extra_hosts', - 'hosts': 'extra_hosts', - 'extra_host': 'extra_hosts', - 'device': 'devices', - 'link': 'links', - 'memory_swap': 'memswap_limit', - 'port': 'ports', - 'privilege': 'privileged', - 'priviliged': 'privileged', - 'privilige': 'privileged', - 'volume': 'volumes', - 'workdir': 'working_dir', -} - SUPPORTED_FILENAMES = [ 'docker-compose.yml', @@ -135,122 +115,18 @@ def get_config_path(base_dir): return os.path.join(path, winner) -@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted.")) -def format_ports(instance): - def _is_valid(port): - if ':' in port or '/' in port: - return True - try: - int(port) - return True - except ValueError: - return False - return False - - if isinstance(instance, list): - for port in instance: - if not _is_valid(port): - return False - return True - elif isinstance(instance, str): - return _is_valid(instance) - return False - - -def get_unsupported_config_msg(service_name, error_key): - msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) - if error_key in DOCKER_CONFIG_HINTS: - msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) - return msg - - -def process_errors(errors): - """ - jsonschema gives us an error tree full of information to explain what has - gone wrong. Process each error and pull out relevant information and re-write - helpful error messages that are relevant. - """ - def _parse_key_from_error_msg(error): - return error.message.split("'")[1] - - root_msgs = [] - invalid_keys = [] - required = [] - type_errors = [] - - for error in errors: - # handle root level errors - if len(error.path) == 0: - if error.validator == 'type': - msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - root_msgs.append(msg) - elif error.validator == 'additionalProperties': - invalid_service_name = _parse_key_from_error_msg(error) - msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS) - root_msgs.append(msg) - else: - root_msgs.append(error.message) - - else: - # handle service level errors - service_name = error.path[0] - - if error.validator == 'additionalProperties': - invalid_config_key = _parse_key_from_error_msg(error) - invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) - elif error.validator == 'anyOf': - if 'image' in error.instance and 'build' in error.instance: - required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name)) - elif 'image' not in error.instance and 'build' not in error.instance: - required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) - else: - required.append(error.message) - elif error.validator == 'type': - msg = "a" - if error.validator_value == "array": - msg = "an" - - try: - config_key = error.path[1] - type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value)) - except IndexError: - config_key = error.path[0] - root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key)) - elif error.validator == 'required': - config_key = error.path[1] - required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message)) - elif error.validator == 'dependencies': - dependency_key = error.validator_value.keys()[0] - required_keys = ",".join(error.validator_value[dependency_key]) - required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( - dependency_key, service_name, dependency_key, required_keys)) - - return "\n".join(root_msgs + invalid_keys + required + type_errors) - - -def validate_against_schema(config): - config_source_dir = os.path.dirname(os.path.abspath(__file__)) - schema_file = os.path.join(config_source_dir, "schema.json") - - with open(schema_file, "r") as schema_fh: - schema = json.load(schema_fh) - - validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"])) - - errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] - if errors: - error_msg = process_errors(errors) - raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg)) - - def load(config_details): config, working_dir, filename = config_details + if not isinstance(config, dict): + raise ConfigurationError( + "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." + ) + config = interpolate_environment_variables(config) + validate_against_schema(config) service_dicts = [] - validate_against_schema(config) - for service_name, service_dict in list(config.items()): loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) @@ -347,13 +223,6 @@ def validate_extended_service_dict(service_dict, filename, service): def process_container_options(service_dict, working_dir=None): - for k in service_dict: - if k not in ALLOWED_KEYS: - msg = "Unsupported config option for %s service: '%s'" % (service_dict['name'], k) - if k in DOCKER_CONFIG_HINTS: - msg += " (did you mean '%s'?)" % DOCKER_CONFIG_HINTS[k] - raise ConfigurationError(msg) - service_dict = service_dict.copy() if 'volumes' in service_dict and service_dict.get('volume_driver') is None: diff --git a/compose/schema.json b/compose/config/schema.json similarity index 100% rename from compose/schema.json rename to compose/config/schema.json diff --git a/compose/config/validation.py b/compose/config/validation.py new file mode 100644 index 0000000000..ba5803decb --- /dev/null +++ b/compose/config/validation.py @@ -0,0 +1,134 @@ +import os + +import json +from jsonschema import Draft4Validator, FormatChecker, ValidationError + +from .errors import ConfigurationError + + +DOCKER_CONFIG_HINTS = { + 'cpu_share': 'cpu_shares', + 'add_host': 'extra_hosts', + 'hosts': 'extra_hosts', + 'extra_host': 'extra_hosts', + 'device': 'devices', + 'link': 'links', + 'memory_swap': 'memswap_limit', + 'port': 'ports', + 'privilege': 'privileged', + 'priviliged': 'privileged', + 'privilige': 'privileged', + 'volume': 'volumes', + 'workdir': 'working_dir', +} + + +VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' + + +@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted.")) +def format_ports(instance): + def _is_valid(port): + if ':' in port or '/' in port: + return True + try: + int(port) + return True + except ValueError: + return False + return False + + if isinstance(instance, list): + for port in instance: + if not _is_valid(port): + return False + return True + elif isinstance(instance, str): + return _is_valid(instance) + return False + + +def get_unsupported_config_msg(service_name, error_key): + msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) + if error_key in DOCKER_CONFIG_HINTS: + msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) + return msg + + +def process_errors(errors): + """ + jsonschema gives us an error tree full of information to explain what has + gone wrong. Process each error and pull out relevant information and re-write + helpful error messages that are relevant. + """ + def _parse_key_from_error_msg(error): + return error.message.split("'")[1] + + root_msgs = [] + invalid_keys = [] + required = [] + type_errors = [] + + for error in errors: + # handle root level errors + if len(error.path) == 0: + if error.validator == 'type': + msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." + root_msgs.append(msg) + elif error.validator == 'additionalProperties': + invalid_service_name = _parse_key_from_error_msg(error) + msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS) + root_msgs.append(msg) + else: + root_msgs.append(error.message) + + else: + # handle service level errors + service_name = error.path[0] + + if error.validator == 'additionalProperties': + invalid_config_key = _parse_key_from_error_msg(error) + invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) + elif error.validator == 'anyOf': + if 'image' in error.instance and 'build' in error.instance: + required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name)) + elif 'image' not in error.instance and 'build' not in error.instance: + required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) + else: + required.append(error.message) + elif error.validator == 'type': + msg = "a" + if error.validator_value == "array": + msg = "an" + + try: + config_key = error.path[1] + type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value)) + except IndexError: + config_key = error.path[0] + root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key)) + elif error.validator == 'required': + config_key = error.path[1] + required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message)) + elif error.validator == 'dependencies': + dependency_key = error.validator_value.keys()[0] + required_keys = ",".join(error.validator_value[dependency_key]) + required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( + dependency_key, service_name, dependency_key, required_keys)) + + return "\n".join(root_msgs + invalid_keys + required + type_errors) + + +def validate_against_schema(config): + config_source_dir = os.path.dirname(os.path.abspath(__file__)) + schema_file = os.path.join(config_source_dir, "schema.json") + + with open(schema_file, "r") as schema_fh: + schema = json.load(schema_fh) + + validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"])) + + errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] + if errors: + error_msg = process_errors(errors) + raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg)) diff --git a/compose/service.py b/compose/service.py index 103840c3be..9b5e5928b9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -12,7 +12,7 @@ from docker.errors import APIError from docker.utils import create_host_config, LogConfig from . import __version__ -from .config import DOCKER_CONFIG_KEYS, merge_environment, VALID_NAME_CHARS +from .config import DOCKER_CONFIG_KEYS, merge_environment from .const import ( DEFAULT_TIMEOUT, LABEL_CONTAINER_NUMBER, @@ -26,6 +26,7 @@ from .container import Container from .legacy import check_for_legacy_containers from .progress_stream import stream_output, StreamOutputError from .utils import json_hash, parallel_execute +from .config.validation import VALID_NAME_CHARS log = logging.getLogger(__name__) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 15657f878f..9f690f111e 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -5,6 +5,7 @@ import tempfile from .. import unittest from compose.config import config +from compose.config.errors import ConfigurationError def make_service_dict(name, service_dict, working_dir): @@ -43,7 +44,7 @@ class ConfigTest(unittest.TestCase): ) def test_load_throws_error_when_not_dict(self): - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(ConfigurationError): config.load( config.ConfigDetails( {'web': 'busybox:latest'}, @@ -52,15 +53,8 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_validation(self): - self.assertRaises( - config.ConfigurationError, - lambda: make_service_dict('foo', {'port': ['8000']}, 'tests/') - ) - make_service_dict('foo', {'ports': ['8000']}, 'tests/') - def test_config_invalid_service_names(self): - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(ConfigurationError): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: config.load( config.ConfigDetails( @@ -81,7 +75,7 @@ class ConfigTest(unittest.TestCase): ) def test_config_invalid_ports_format_validation(self): - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(ConfigurationError): for invalid_ports in [{"1": "8000"}, "whatport"]: config.load( config.ConfigDetails( @@ -104,7 +98,7 @@ class ConfigTest(unittest.TestCase): def test_config_hint(self): expected_error_msg = "(did you mean 'privileged'?)" - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -117,7 +111,7 @@ class ConfigTest(unittest.TestCase): def test_invalid_config_build_and_image_specified(self): expected_error_msg = "Service 'foo' has both an image and build path specified." - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -130,7 +124,7 @@ class ConfigTest(unittest.TestCase): def test_invalid_config_type_should_be_an_array(self): expected_error_msg = "Service 'foo' has an invalid value for 'links', it should be an array" - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -143,7 +137,7 @@ class ConfigTest(unittest.TestCase): def test_invalid_config_not_a_dictionary(self): expected_error_msg = "Top level object needs to be a dictionary." - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( ['foo', 'lol'], @@ -198,7 +192,7 @@ class InterpolationTest(unittest.TestCase): os.environ['VOLUME_PATH'] = '/host/path' d = config.load( config.ConfigDetails( - config={'foo': {'volumes': ['${VOLUME_PATH}:/container/path']}}, + config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, working_dir='.', filename=None, ) @@ -422,7 +416,7 @@ class MemoryOptionsTest(unittest.TestCase): a mem_limit """ expected_error_msg = "Invalid 'memswap_limit' configuration for 'foo' service: when defining 'memswap_limit' you must set 'mem_limit' as well" - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -465,7 +459,7 @@ class EnvTest(unittest.TestCase): self.assertEqual(config.parse_environment(environment), environment) def test_parse_environment_invalid(self): - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(ConfigurationError): config.parse_environment('a=b') def test_parse_environment_empty(self): @@ -519,7 +513,7 @@ class EnvTest(unittest.TestCase): def test_env_nonexistent_file(self): options = {'env_file': 'nonexistent.env'} self.assertRaises( - config.ConfigurationError, + ConfigurationError, lambda: make_service_dict('foo', options, 'tests/fixtures/env'), ) @@ -545,7 +539,7 @@ class EnvTest(unittest.TestCase): service_dict = config.load( config.ConfigDetails( - config={'foo': {'volumes': ['$HOSTENV:$CONTAINERENV']}}, + config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, working_dir="tests/fixtures/env", filename=None, ) @@ -554,7 +548,7 @@ class EnvTest(unittest.TestCase): service_dict = config.load( config.ConfigDetails( - config={'foo': {'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, + config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, working_dir="tests/fixtures/env", filename=None, ) @@ -652,7 +646,7 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_empty_dictionary(self): - with self.assertRaisesRegexp(config.ConfigurationError, 'service'): + with self.assertRaisesRegexp(ConfigurationError, 'service'): config.load( config.ConfigDetails( { @@ -664,7 +658,7 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_missing_service_key(self): - with self.assertRaisesRegexp(config.ConfigurationError, "u'service' is a required property"): + with self.assertRaisesRegexp(ConfigurationError, "u'service' is a required property"): config.load( config.ConfigDetails( { @@ -677,7 +671,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_invalid_key(self): expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" - with self.assertRaisesRegexp(config.ConfigurationError, expected_error_msg): + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( { @@ -701,7 +695,7 @@ class ExtendsTest(unittest.TestCase): def load_config(): return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - self.assertRaisesRegexp(config.ConfigurationError, 'file', load_config) + self.assertRaisesRegexp(ConfigurationError, 'file', load_config) def test_extends_validation_valid_config(self): service = config.load( @@ -750,19 +744,19 @@ class ExtendsTest(unittest.TestCase): } }, '.') - with self.assertRaisesRegexp(config.ConfigurationError, 'links'): + with self.assertRaisesRegexp(ConfigurationError, 'links'): other_config = {'web': {'links': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): print load_config() - with self.assertRaisesRegexp(config.ConfigurationError, 'volumes_from'): + with self.assertRaisesRegexp(ConfigurationError, 'volumes_from'): other_config = {'web': {'volumes_from': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): print load_config() - with self.assertRaisesRegexp(config.ConfigurationError, 'net'): + with self.assertRaisesRegexp(ConfigurationError, 'net'): other_config = {'web': {'net': 'container:db'}} with mock.patch.object(config, 'load_yaml', return_value=other_config): @@ -804,7 +798,7 @@ class BuildPathTest(unittest.TestCase): self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') def test_nonexistent_path(self): - with self.assertRaises(config.ConfigurationError): + with self.assertRaises(ConfigurationError): config.load( config.ConfigDetails( { From df74b131ff8ca8f6055a1e16d4e9c7ff56462370 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 7 Aug 2015 15:26:26 +0100 Subject: [PATCH 031/337] Use split_port for ports format check Rather than implement the logic a second time, use docker-py split_port function to test if the ports is valid. Signed-off-by: Mazz Mosley --- compose/config/schema.json | 13 ++++--------- compose/config/validation.py | 24 ++++++------------------ tests/unit/config_test.py | 4 ++-- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 258f44ccac..74f5edbbff 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -75,15 +75,10 @@ "pid": {"type": "string"}, "ports": { - "oneOf": [ - {"type": "string", "format": "ports"}, - { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true, - "format": "ports" - } - ] + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true, + "format": "ports" }, "privileged": {"type": "string"}, diff --git a/compose/config/validation.py b/compose/config/validation.py index ba5803decb..15e0754cf8 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,5 +1,6 @@ import os +from docker.utils.ports import split_port import json from jsonschema import Draft4Validator, FormatChecker, ValidationError @@ -26,26 +27,13 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' -@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted.")) +@FormatChecker.cls_checks(format="ports", raises=ValidationError("Invalid port formatting, it should be '[[remote_ip:]remote_port:]port[/protocol]'")) def format_ports(instance): - def _is_valid(port): - if ':' in port or '/' in port: - return True - try: - int(port) - return True - except ValueError: - return False + try: + split_port(instance) + except ValueError: return False - - if isinstance(instance, list): - for port in instance: - if not _is_valid(port): - return False - return True - elif isinstance(instance, str): - return _is_valid(instance) - return False + return True def get_unsupported_config_msg(service_name, error_key): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 9f690f111e..4e982bb49a 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -76,7 +76,7 @@ class ConfigTest(unittest.TestCase): def test_config_invalid_ports_format_validation(self): with self.assertRaises(ConfigurationError): - for invalid_ports in [{"1": "8000"}, "whatport"]: + for invalid_ports in [{"1": "8000"}, "whatport", "625", "8000:8050"]: config.load( config.ConfigDetails( {'web': {'image': 'busybox', 'ports': invalid_ports}}, @@ -86,7 +86,7 @@ class ConfigTest(unittest.TestCase): ) def test_config_valid_ports_format_validation(self): - valid_ports = [["8000", "9000"], "625", "8000:8050", ["8000/8050"]] + valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"]] for ports in valid_ports: config.load( config.ConfigDetails( From 297941e460bbd1556e4fa5b81ee5816f3f4b0773 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Mon, 6 Jul 2015 12:15:50 -0400 Subject: [PATCH 032/337] rebasing port range changes Signed-off-by: Yuval Kohavi --- compose/service.py | 47 ++----- .../ports-composefile/docker-compose.yml | 1 + tests/integration/cli_test.py | 5 +- tests/unit/service_test.py | 127 ++++-------------- 4 files changed, 40 insertions(+), 140 deletions(-) diff --git a/compose/service.py b/compose/service.py index 2e0490a508..07f268c267 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ from operator import attrgetter import six from docker.errors import APIError from docker.utils import create_host_config, LogConfig +from docker.utils.ports import build_port_bindings, split_port from . import __version__ from .config import DOCKER_CONFIG_KEYS, merge_environment @@ -599,13 +600,13 @@ class Service(object): if 'ports' in container_options or 'expose' in self.options: ports = [] all_ports = container_options.get('ports', []) + self.options.get('expose', []) - for port in all_ports: - port = str(port) - if ':' in port: - port = port.split(':')[-1] - if '/' in port: - port = tuple(port.split('/')) - ports.append(port) + for port_range in all_ports: + internal_range, _ = split_port(port_range) + for port in internal_range: + port = str(port) + if '/' in port: + port = tuple(port.split('/')) + ports.append(port) container_options['ports'] = ports override_options['binds'] = merge_volume_bindings( @@ -859,38 +860,6 @@ def parse_volume_spec(volume_config): return VolumeSpec(external, internal, mode) - -# Ports - - -def build_port_bindings(ports): - port_bindings = {} - for port in ports: - internal_port, external = split_port(port) - if internal_port in port_bindings: - port_bindings[internal_port].append(external) - else: - port_bindings[internal_port] = [external] - return port_bindings - - -def split_port(port): - parts = str(port).split(':') - if not 1 <= len(parts) <= 3: - raise ConfigError('Invalid port "%s", should be ' - '[[remote_ip:]remote_port:]port[/protocol]' % port) - - if len(parts) == 1: - internal_port, = parts - return internal_port, None - if len(parts) == 2: - external_port, internal_port = parts - return internal_port, external_port - - external_ip, external_port, internal_port = parts - return internal_port, (external_ip, external_port or None) - - # Labels diff --git a/tests/fixtures/ports-composefile/docker-compose.yml b/tests/fixtures/ports-composefile/docker-compose.yml index 9496ee0826..c213068def 100644 --- a/tests/fixtures/ports-composefile/docker-compose.yml +++ b/tests/fixtures/ports-composefile/docker-compose.yml @@ -5,3 +5,4 @@ simple: ports: - '3000' - '49152:3001' + - '49153-49154:3002-3003' diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 0e86c2792f..e844fa2a3b 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -334,6 +334,7 @@ class CLITestCase(DockerClientTestCase): # get port information port_random = container.get_local_port(3000) port_assigned = container.get_local_port(3001) + port_range = container.get_local_port(3002), container.get_local_port(3003) # close all one off containers we just created container.stop() @@ -342,6 +343,8 @@ class CLITestCase(DockerClientTestCase): self.assertNotEqual(port_random, None) self.assertIn("0.0.0.0", port_random) self.assertEqual(port_assigned, "0.0.0.0:49152") + self.assertEqual(port_range[0], "0.0.0.0:49153") + self.assertEqual(port_range[1], "0.0.0.0:49154") def test_rm(self): service = self.project.get_service('simple') @@ -456,7 +459,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3000), container.get_local_port(3000)) self.assertEqual(get_port(3001), "0.0.0.0:49152") - self.assertEqual(get_port(3002), "") + self.assertEqual(get_port(3002), "0.0.0.0:49153") def test_port_with_scale(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index bc6b9e485e..151fcee94b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -5,7 +5,6 @@ from .. import unittest import mock import docker -from docker.utils import LogConfig from compose.service import Service from compose.container import Container @@ -13,14 +12,11 @@ from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF from compose.service import ( ConfigError, NeedsBuildError, - NoSuchImageError, - build_port_bindings, build_volume_binding, get_container_data_volumes, merge_volume_bindings, parse_repository_tag, parse_volume_spec, - split_port, ) @@ -108,48 +104,6 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_volumes_from(), [container_id]) from_service.create_container.assert_called_once_with() - def test_split_port_with_host_ip(self): - internal_port, external_port = split_port("127.0.0.1:1000:2000") - self.assertEqual(internal_port, "2000") - self.assertEqual(external_port, ("127.0.0.1", "1000")) - - def test_split_port_with_protocol(self): - internal_port, external_port = split_port("127.0.0.1:1000:2000/udp") - self.assertEqual(internal_port, "2000/udp") - self.assertEqual(external_port, ("127.0.0.1", "1000")) - - def test_split_port_with_host_ip_no_port(self): - internal_port, external_port = split_port("127.0.0.1::2000") - self.assertEqual(internal_port, "2000") - self.assertEqual(external_port, ("127.0.0.1", None)) - - def test_split_port_with_host_port(self): - internal_port, external_port = split_port("1000:2000") - self.assertEqual(internal_port, "2000") - self.assertEqual(external_port, "1000") - - def test_split_port_no_host_port(self): - internal_port, external_port = split_port("2000") - self.assertEqual(internal_port, "2000") - self.assertEqual(external_port, None) - - def test_split_port_invalid(self): - with self.assertRaises(ConfigError): - split_port("0.0.0.0:1000:2000:tcp") - - def test_build_port_bindings_with_one_port(self): - port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) - - def test_build_port_bindings_with_matching_internal_ports(self): - port_bindings = build_port_bindings(["127.0.0.1:1000:1000", "127.0.0.1:2000:1000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000"), ("127.0.0.1", "2000")]) - - def test_build_port_bindings_with_nonmatching_internal_ports(self): - port_bindings = build_port_bindings(["127.0.0.1:1000:1000", "127.0.0.1:2000:2000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) - self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) - def test_split_domainname_none(self): service = Service('foo', image='foo', hostname='name', client=self.mock_client) self.mock_client.containers.return_value = [] @@ -157,23 +111,6 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertFalse('domainname' in opts, 'domainname') - def test_memory_swap_limit(self): - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) - self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'some': 'overrides'}, 1) - self.assertEqual(opts['memswap_limit'], 2000000000) - self.assertEqual(opts['mem_limit'], 1000000000) - - def test_log_opt(self): - log_opt = {'address': 'tcp://192.168.0.42:123'} - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) - self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'some': 'overrides'}, 1) - - self.assertIsInstance(opts['host_config']['LogConfig'], LogConfig) - self.assertEqual(opts['host_config']['LogConfig'].type, 'syslog') - self.assertEqual(opts['host_config']['LogConfig'].config, log_opt) - def test_split_domainname_fqdn(self): service = Service( 'foo', @@ -229,10 +166,11 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.log', autospec=True) def test_pull_image(self, mock_log): service = Service('foo', client=self.mock_client, image='someimage:sometag') - service.pull() + service.pull(insecure_registry=True) self.mock_client.pull.assert_called_once_with( 'someimage', tag='sometag', + insecure_registry=True, stream=True) mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...') @@ -242,8 +180,26 @@ class ServiceTest(unittest.TestCase): self.mock_client.pull.assert_called_once_with( 'ababab', tag='latest', + insecure_registry=False, stream=True) + def test_create_container_from_insecure_registry(self): + service = Service('foo', client=self.mock_client, image='someimage:sometag') + images = [] + + def pull(repo, tag=None, insecure_registry=False, **kwargs): + self.assertEqual('someimage', repo) + self.assertEqual('sometag', tag) + self.assertTrue(insecure_registry) + images.append({'Id': 'abc123'}) + return [] + + service.image = lambda: images[0] if images else None + self.mock_client.pull = pull + + service.create_container(insecure_registry=True) + self.assertEqual(1, len(images)) + @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) @@ -287,7 +243,7 @@ class ServiceTest(unittest.TestCase): images.append({'Id': 'abc123'}) return [] - service.image = lambda *args, **kwargs: mock_get_image(images) + service.image = lambda: images[0] if images else None self.mock_client.pull = pull service.create_container() @@ -297,7 +253,7 @@ class ServiceTest(unittest.TestCase): service = Service('foo', client=self.mock_client, build='.') images = [] - service.image = lambda *args, **kwargs: mock_get_image(images) + service.image = lambda *args, **kwargs: images[0] if images else None service.build = lambda: images.append({'Id': 'abc123'}) service.create_container(do_build=True) @@ -312,7 +268,7 @@ class ServiceTest(unittest.TestCase): def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') - service.image = lambda *args, **kwargs: mock_get_image([]) + service.image = lambda: None with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -329,13 +285,6 @@ class ServiceTest(unittest.TestCase): self.assertFalse(self.mock_client.build.call_args[1]['pull']) -def mock_get_image(images): - if images: - return images[0] - else: - raise NoSuchImageError() - - class ServiceVolumesTest(unittest.TestCase): def setUp(self): @@ -353,13 +302,14 @@ class ServiceVolumesTest(unittest.TestCase): spec = parse_volume_spec('external:interval:ro') self.assertEqual(spec, ('external', 'interval', 'ro')) - spec = parse_volume_spec('external:interval:z') - self.assertEqual(spec, ('external', 'interval', 'z')) - def test_parse_volume_spec_too_many_parts(self): with self.assertRaises(ConfigError): parse_volume_spec('one:two:three:four') + def test_parse_volume_bad_mode(self): + with self.assertRaises(ConfigError): + parse_volume_spec('one:two:notrw') + def test_build_volume_binding(self): binding = build_volume_binding(parse_volume_spec('/outside:/inside')) self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) @@ -488,26 +438,3 @@ class ServiceVolumesTest(unittest.TestCase): create_options['host_config']['Binds'], ['/mnt/sda1/host/path:/data:rw'], ) - - def test_create_with_special_volume_mode(self): - self.mock_client.inspect_image.return_value = {'Id': 'imageid'} - - create_calls = [] - - def create_container(*args, **kwargs): - create_calls.append((args, kwargs)) - return {'Id': 'containerid'} - - self.mock_client.create_container = create_container - - volumes = ['/tmp:/foo:z'] - - Service( - 'web', - client=self.mock_client, - image='busybox', - volumes=volumes, - ).create_container() - - self.assertEqual(len(create_calls), 1) - self.assertEqual(create_calls[0][1]['host_config']['Binds'], volumes) From 0fdd977b06f95d3eadf04aa975ade85b8cc00e5f Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 28 Jul 2015 17:00:24 -0400 Subject: [PATCH 033/337] fixed merge issue from previous commit Signed-off-by: Yuval Kohavi --- tests/unit/service_test.py | 83 +++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 151fcee94b..77a8138d0b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -5,6 +5,7 @@ from .. import unittest import mock import docker +from docker.utils import LogConfig from compose.service import Service from compose.container import Container @@ -12,6 +13,7 @@ from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF from compose.service import ( ConfigError, NeedsBuildError, + NoSuchImageError, build_volume_binding, get_container_data_volumes, merge_volume_bindings, @@ -111,6 +113,23 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertFalse('domainname' in opts, 'domainname') + def test_memory_swap_limit(self): + service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) + self.mock_client.containers.return_value = [] + opts = service._get_container_create_options({'some': 'overrides'}, 1) + self.assertEqual(opts['memswap_limit'], 2000000000) + self.assertEqual(opts['mem_limit'], 1000000000) + + def test_log_opt(self): + log_opt = {'address': 'tcp://192.168.0.42:123'} + service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) + self.mock_client.containers.return_value = [] + opts = service._get_container_create_options({'some': 'overrides'}, 1) + + self.assertIsInstance(opts['host_config']['LogConfig'], LogConfig) + self.assertEqual(opts['host_config']['LogConfig'].type, 'syslog') + self.assertEqual(opts['host_config']['LogConfig'].config, log_opt) + def test_split_domainname_fqdn(self): service = Service( 'foo', @@ -166,11 +185,10 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.log', autospec=True) def test_pull_image(self, mock_log): service = Service('foo', client=self.mock_client, image='someimage:sometag') - service.pull(insecure_registry=True) + service.pull() self.mock_client.pull.assert_called_once_with( 'someimage', tag='sometag', - insecure_registry=True, stream=True) mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...') @@ -180,26 +198,8 @@ class ServiceTest(unittest.TestCase): self.mock_client.pull.assert_called_once_with( 'ababab', tag='latest', - insecure_registry=False, stream=True) - def test_create_container_from_insecure_registry(self): - service = Service('foo', client=self.mock_client, image='someimage:sometag') - images = [] - - def pull(repo, tag=None, insecure_registry=False, **kwargs): - self.assertEqual('someimage', repo) - self.assertEqual('sometag', tag) - self.assertTrue(insecure_registry) - images.append({'Id': 'abc123'}) - return [] - - service.image = lambda: images[0] if images else None - self.mock_client.pull = pull - - service.create_container(insecure_registry=True) - self.assertEqual(1, len(images)) - @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) @@ -243,7 +243,7 @@ class ServiceTest(unittest.TestCase): images.append({'Id': 'abc123'}) return [] - service.image = lambda: images[0] if images else None + service.image = lambda *args, **kwargs: mock_get_image(images) self.mock_client.pull = pull service.create_container() @@ -253,7 +253,7 @@ class ServiceTest(unittest.TestCase): service = Service('foo', client=self.mock_client, build='.') images = [] - service.image = lambda *args, **kwargs: images[0] if images else None + service.image = lambda *args, **kwargs: mock_get_image(images) service.build = lambda: images.append({'Id': 'abc123'}) service.create_container(do_build=True) @@ -268,7 +268,7 @@ class ServiceTest(unittest.TestCase): def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') - service.image = lambda: None + service.image = lambda *args, **kwargs: mock_get_image([]) with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -285,6 +285,13 @@ class ServiceTest(unittest.TestCase): self.assertFalse(self.mock_client.build.call_args[1]['pull']) +def mock_get_image(images): + if images: + return images[0] + else: + raise NoSuchImageError() + + class ServiceVolumesTest(unittest.TestCase): def setUp(self): @@ -302,14 +309,13 @@ class ServiceVolumesTest(unittest.TestCase): spec = parse_volume_spec('external:interval:ro') self.assertEqual(spec, ('external', 'interval', 'ro')) + spec = parse_volume_spec('external:interval:z') + self.assertEqual(spec, ('external', 'interval', 'z')) + def test_parse_volume_spec_too_many_parts(self): with self.assertRaises(ConfigError): parse_volume_spec('one:two:three:four') - def test_parse_volume_bad_mode(self): - with self.assertRaises(ConfigError): - parse_volume_spec('one:two:notrw') - def test_build_volume_binding(self): binding = build_volume_binding(parse_volume_spec('/outside:/inside')) self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) @@ -438,3 +444,26 @@ class ServiceVolumesTest(unittest.TestCase): create_options['host_config']['Binds'], ['/mnt/sda1/host/path:/data:rw'], ) + + def test_create_with_special_volume_mode(self): + self.mock_client.inspect_image.return_value = {'Id': 'imageid'} + + create_calls = [] + + def create_container(*args, **kwargs): + create_calls.append((args, kwargs)) + return {'Id': 'containerid'} + + self.mock_client.create_container = create_container + + volumes = ['/tmp:/foo:z'] + + Service( + 'web', + client=self.mock_client, + image='busybox', + volumes=volumes, + ).create_container() + + self.assertEqual(len(create_calls), 1) + self.assertEqual(create_calls[0][1]['host_config']['Binds'], volumes) From 557cbb616c887d35c0c79e2ed2a79c0344f58de4 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 28 Jul 2015 17:26:32 -0400 Subject: [PATCH 034/337] ports documentation Signed-off-by: Yuval Kohavi --- docs/yml.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/yml.md b/docs/yml.md index 18551bf22f..17dbc59ad2 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -106,7 +106,7 @@ An entry with the ip address and hostname will be created in `/etc/hosts` inside ### ports Expose ports. Either specify both ports (`HOST:CONTAINER`), or just the container -port (a random host port will be chosen). +port (a random host port will be chosen). You can specify a port range instead of a single port (`START-END`). If you use a range for the container ports, you may specify a range for the host ports as well. both ranges must be of equal size. > **Note:** When mapping ports in the `HOST:CONTAINER` format, you may experience > erroneous results when using a container port lower than 60, because YAML will @@ -115,9 +115,12 @@ port (a random host port will be chosen). ports: - "3000" + - "3000-3005" - "8000:8000" + - "9090-9091:8080-8081" - "49100:22" - "127.0.0.1:8001:8001" + - "127.0.0.1:5000-5010:5000-5010" ### expose @@ -410,3 +413,4 @@ dollar sign (`$$`). - [Command line reference](cli.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) + From d1455acb6469f1a36376f5f765ed5188336673ee Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 7 Aug 2015 16:30:00 +0100 Subject: [PATCH 035/337] Update docs inline with feedback Signed-off-by: Mazz Mosley --- docs/yml.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 17dbc59ad2..1055004254 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -105,13 +105,25 @@ An entry with the ip address and hostname will be created in `/etc/hosts` inside ### ports -Expose ports. Either specify both ports (`HOST:CONTAINER`), or just the container -port (a random host port will be chosen). You can specify a port range instead of a single port (`START-END`). If you use a range for the container ports, you may specify a range for the host ports as well. both ranges must be of equal size. +Makes an exposed port accessible on a host and the port is available to +any client that can reach that host. Docker binds the exposed port to a random +port on the host within an *ephemeral port range* defined by +`/proc/sys/net/ipv4/ip_local_port_range`. You can also map to a specific port or range of ports. -> **Note:** When mapping ports in the `HOST:CONTAINER` format, you may experience -> erroneous results when using a container port lower than 60, because YAML will -> parse numbers in the format `xx:yy` as sexagesimal (base 60). For this reason, -> we recommend always explicitly specifying your port mappings as strings. +Acceptable formats for the `ports` value are: + +* `containerPort` +* `ip:hostPort:containerPort` +* `ip::containerPort` +* `hostPort:containerPort` + +You can specify a range for both the `hostPort` and the `containerPort` values. +When specifying ranges, the container port values in the range must match the +number of host port values in the range, for example, +`1234-1236:1234-1236/tcp`. Once a host is running, use the 'docker-compose port' command +to see the actual mapping. + +The following configuration shows examples of the port formats in use: ports: - "3000" @@ -122,6 +134,13 @@ port (a random host port will be chosen). You can specify a port range instead o - "127.0.0.1:8001:8001" - "127.0.0.1:5000-5010:5000-5010" + +When mapping ports, in the `hostPort:containerPort` format, you may +experience erroneous results when using a container port lower than 60. This +happens because YAML parses numbers in the format `xx:yy` as sexagesimal (base +60). To avoid this problem, always explicitly specify your port +mappings as strings. + ### expose Expose ports without publishing them to the host machine - they'll only be From 11adca9324b417729d8eafdab1852767455f8cca Mon Sep 17 00:00:00 2001 From: Veres Lajos Date: Fri, 7 Aug 2015 21:59:14 +0100 Subject: [PATCH 036/337] typofix - https://github.com/vlajos/misspell_fixer Signed-off-by: Veres Lajos --- tests/unit/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0046202030..2cd3e00559 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -9,7 +9,7 @@ from compose.config import config def make_service_dict(name, service_dict, working_dir): """ - Test helper function to contruct a ServiceLoader + Test helper function to construct a ServiceLoader """ return config.ServiceLoader(working_dir=working_dir).make_service_dict(name, service_dict) From 7c128b46a1daff8e333ecdd611eaf0c6e42bb197 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Wed, 5 Aug 2015 10:38:40 -0700 Subject: [PATCH 037/337] - Closes #1811 for Toolbox - Updating with comments Signed-off-by: Mary Anthony --- docs/install.md | 78 +++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/docs/install.md b/docs/install.md index adb32fd50b..fa36791927 100644 --- a/docs/install.md +++ b/docs/install.md @@ -12,50 +12,67 @@ weight=4 # Install Docker Compose -To install Compose, you'll need to install Docker first. You'll then install -Compose with a `curl` command. +You can run Compose on OS X and 64-bit Linux. It is currently not supported on +the Windows operating system. To install Compose, you'll need to install Docker +first. -## Install Docker +Depending on how your system is configured, you may require `sudo` access to +install Compose. If your system requires `sudo`, you will receive "Permission +denied" errors when installing Compose. If this is the case for you, preface the +install commands with `sudo` to install. -First, install Docker version 1.7.1 or greater: +To install Compose, do the following: -- [Instructions for Mac OS X](http://docs.docker.com/installation/mac/) -- [Instructions for Ubuntu](http://docs.docker.com/installation/ubuntulinux/) -- [Instructions for other systems](http://docs.docker.com/installation/) +1. Install Docker Engine version 1.7.1 or greater: -## Install Compose + * Mac OS X installation (installs both Engine and Compose) + + * Ubuntu installation + + * other system installations + +2. Mac OS X users are done installing. Others should continue to the next step. + +3. Go to the repository release page. -To install Compose, run the following commands: +4. Enter the `curl` command in your termial. - curl -L https://github.com/docker/compose/releases/download/1.3.3/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + The command has the following format: -> Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. + curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + + If you have problems installing with `curl`, you can use `pip` instead: `pip install -U docker-compose` + +4. Apply executable permissions to the binary: -Optionally, you can also install [command completion](completion.md) for the -bash and zsh shell. + $ chmod +x /usr/local/bin/docker-compose -> **Note:** Some older Mac OS X CPU architectures are incompatible with the binary. If you receive an "Illegal instruction: 4" error after installing, you should install using the `pip` command instead. +5. Optionally, install [command completion](completion.md) for the +`bash` and `zsh` shell. -Compose is available for OS X and 64-bit Linux. If you're on another platform, -Compose can also be installed as a Python package: +6. Test the installation. - $ sudo pip install -U docker-compose + $ docker-compose --version + docker-compose version: 1.4.0 -No further steps are required; Compose should now be successfully installed. -You can test the installation by running `docker-compose --version`. +## Upgrading -### Upgrading +If you're upgrading from Compose 1.2 or earlier, you'll need to remove or migrate +your existing containers after upgrading Compose. This is because, as of version +1.3, Compose uses Docker labels to keep track of containers, and so they need to +be recreated with labels added. -If you're coming from Compose 1.2 or earlier, you'll need to remove or migrate your existing containers after upgrading Compose. This is because, as of version 1.3, Compose uses Docker labels to keep track of containers, and so they need to be recreated with labels added. +If Compose detects containers that were created without labels, it will refuse +to run so that you don't end up with two sets of them. If you want to keep using +your existing containers (for example, because they have data volumes you want +to preserve) you can migrate them with the following command: -If Compose detects containers that were created without labels, it will refuse to run so that you don't end up with two sets of them. If you want to keep using your existing containers (for example, because they have data volumes you want to preserve) you can migrate them with the following command: + $ docker-compose migrate-to-labels - docker-compose migrate-to-labels +Alternatively, if you're not worried about keeping them, you can remove them &endash; +Compose will just create new ones. -Alternatively, if you're not worried about keeping them, you can remove them - Compose will just create new ones. - - docker rm -f myapp_web_1 myapp_db_1 ... + $ docker rm -f -v myapp_web_1 myapp_db_1 ... ## Uninstallation @@ -69,10 +86,13 @@ To uninstall Docker Compose if you installed using `pip`: $ pip uninstall docker-compose -> Note: If you get a "Permission denied" error using either of the above methods, you probably do not have the proper permissions to remove `docker-compose`. To force the removal, prepend `sudo` to either of the above commands and run again. +>**Note**: If you get a "Permission denied" error using either of the above +>methods, you probably do not have the proper permissions to remove +>`docker-compose`. To force the removal, prepend `sudo` to either of the above +>commands and run again. -## Compose documentation +## Where to go next - [User guide](/) - [Get started with Django](django.md) From 4390362366babb04b0b68759814206d2faff2b63 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 5 Aug 2015 15:39:07 +0100 Subject: [PATCH 038/337] Test against Docker 1.8.0 RC3 Signed-off-by: Aanand Prasad --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0e7f14f91..7c0482323b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,11 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.7.1 +ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.0-rc3 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.7.1 -o /usr/local/bin/docker-1.7.1; \ - chmod +x /usr/local/bin/docker-1.7.1 + chmod +x /usr/local/bin/docker-1.7.1; \ + curl https://test.docker.com/builds/Linux/x86_64/docker-1.8.0-rc3 -o /usr/local/bin/docker-1.8.0-rc3; \ + chmod +x /usr/local/bin/docker-1.8.0-rc3 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker From dfa4bf4452584f1a2533254231b862d521d75f85 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 10 Aug 2015 16:00:45 +0100 Subject: [PATCH 039/337] Ignore containers that don't have a name If a container is in the process of being removed, or removal has failed, it can sometimes appear in the output of GET /containers/json but not have a 'Name' key. In that case, rather than crashing, we can ignore it. Signed-off-by: Aanand Prasad --- compose/container.py | 6 +++++- compose/project.py | 4 ++-- compose/service.py | 11 ++++++----- tests/integration/legacy_test.py | 17 ++++++++++++++--- tests/unit/project_test.py | 25 +++++++++++++++++++++++++ tests/unit/service_test.py | 12 ++++++++++++ 6 files changed, 64 insertions(+), 11 deletions(-) diff --git a/compose/container.py b/compose/container.py index 7195149716..40aea98a45 100644 --- a/compose/container.py +++ b/compose/container.py @@ -22,10 +22,14 @@ class Container(object): """ Construct a container object from the output of GET /containers/json. """ + name = get_container_name(dictionary) + if name is None: + return None + new_dictionary = { 'Id': dictionary['Id'], 'Image': dictionary['Image'], - 'Name': '/' + get_container_name(dictionary), + 'Name': '/' + name, } return cls(client, new_dictionary, **kwargs) diff --git a/compose/project.py b/compose/project.py index 2667855d9c..6d86a4a872 100644 --- a/compose/project.py +++ b/compose/project.py @@ -310,11 +310,11 @@ class Project(object): else: service_names = self.service_names - containers = [ + containers = filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters={'label': self.labels(one_off=one_off)})] + filters={'label': self.labels(one_off=one_off)})]) def matches_service_names(container): return container.labels.get(LABEL_SERVICE) in service_names diff --git a/compose/service.py b/compose/service.py index 2e0490a508..2cdd6c9b58 100644 --- a/compose/service.py +++ b/compose/service.py @@ -101,11 +101,11 @@ class Service(object): self.options = options def containers(self, stopped=False, one_off=False): - containers = [ + containers = filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters={'label': self.labels(one_off=one_off)})] + filters={'label': self.labels(one_off=one_off)})]) if not containers: check_for_legacy_containers( @@ -494,12 +494,13 @@ class Service(object): # TODO: this would benefit from github.com/docker/docker/pull/11943 # to remove the need to inspect every container def _next_container_number(self, one_off=False): - numbers = [ - Container.from_ps(self.client, container).number + containers = filter(None, [ + Container.from_ps(self.client, container) for container in self.client.containers( all=True, filters={'label': self.labels(one_off=one_off)}) - ] + ]) + numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index f79089b207..9913bbb0fe 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -65,7 +65,7 @@ class UtilitiesTestCase(unittest.TestCase): legacy.is_valid_name("composetest_web_lol_1", one_off=True), ) - def test_get_legacy_containers_no_labels(self): + def test_get_legacy_containers(self): client = Mock() client.containers.return_value = [ { @@ -74,12 +74,23 @@ class UtilitiesTestCase(unittest.TestCase): "Name": "composetest_web_1", "Labels": None, }, + { + "Id": "ghi789", + "Image": "def456", + "Name": None, + "Labels": None, + }, + { + "Id": "jkl012", + "Image": "def456", + "Labels": None, + }, ] - containers = list(legacy.get_legacy_containers( - client, "composetest", ["web"])) + containers = legacy.get_legacy_containers(client, "composetest", ["web"]) self.assertEqual(len(containers), 1) + self.assertEqual(containers[0].id, 'abc123') class LegacyTestCase(DockerClientTestCase): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 39ad30a152..93bf12ff57 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -3,6 +3,7 @@ from .. import unittest from compose.service import Service from compose.project import Project from compose.container import Container +from compose.const import LABEL_SERVICE import mock import docker @@ -260,3 +261,27 @@ class ProjectTest(unittest.TestCase): service = project.get_service('test') self.assertEqual(service._get_net(), 'container:' + container_name) + + def test_container_without_name(self): + self.mock_client.containers.return_value = [ + {'Image': 'busybox:latest', 'Id': '1', 'Name': '1'}, + {'Image': 'busybox:latest', 'Id': '2', 'Name': None}, + {'Image': 'busybox:latest', 'Id': '3'}, + ] + self.mock_client.inspect_container.return_value = { + 'Id': '1', + 'Config': { + 'Labels': { + LABEL_SERVICE: 'web', + }, + }, + } + project = Project.from_dicts( + 'test', + [{ + 'name': 'web', + 'image': 'busybox:latest', + }], + self.mock_client, + ) + self.assertEqual([c.id for c in project.containers()], ['1']) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index bc6b9e485e..0e274a3583 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -76,6 +76,18 @@ class ServiceTest(unittest.TestCase): all=False, filters={'label': expected_labels}) + def test_container_without_name(self): + self.mock_client.containers.return_value = [ + {'Image': 'foo', 'Id': '1', 'Name': '1'}, + {'Image': 'foo', 'Id': '2', 'Name': None}, + {'Image': 'foo', 'Id': '3'}, + ] + service = Service('db', self.mock_client, 'myproject', image='foo') + + self.assertEqual([c.id for c in service.containers()], ['1']) + self.assertEqual(service._next_container_number(), 2) + self.assertEqual(service.get_container(1).id, '1') + def test_get_volumes_from_container(self): container_id = 'aabbccddee' service = Service( From 7f90e9592a55f198ecd89799ee2bb7579d5b7aae Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 10 Aug 2015 18:05:09 +0100 Subject: [PATCH 040/337] Use overlay driver in tests Signed-off-by: Aanand Prasad --- script/wrapdocker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/wrapdocker b/script/wrapdocker index 119e88df4a..3e669b5d7a 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -7,7 +7,7 @@ fi # If a pidfile is still around (for example after a container restart), # delete it so that docker can start. rm -rf /var/run/docker.pid -docker -d $DOCKER_DAEMON_ARGS &>/var/log/docker.log & +docker -d --storage-driver="overlay" &>/var/log/docker.log & docker_pid=$! >&2 echo "Waiting for Docker to start..." From 46e8e4322aa694f176c3fec5e705c5c40c704824 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 10 Aug 2015 13:41:11 +0100 Subject: [PATCH 041/337] Show a warning when a relative path is specified without "./" Signed-off-by: Aanand Prasad --- compose/config/config.py | 29 +++++++++++++++++++++---- docs/yml.md | 5 +++-- tests/unit/config_test.py | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 4d3f5faefa..73516a21d5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -90,6 +90,13 @@ SUPPORTED_FILENAMES = [ ] +PATH_START_CHARS = [ + '/', + '.', + '~', +] + + log = logging.getLogger(__name__) @@ -260,7 +267,7 @@ def process_container_options(service_dict, working_dir=None): raise ConfigurationError("Invalid 'memswap_limit' configuration for %s service: when defining 'memswap_limit' you must set 'mem_limit' as well" % service_dict['name']) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: - service_dict['volumes'] = resolve_volume_paths(service_dict['volumes'], working_dir=working_dir) + service_dict['volumes'] = resolve_volume_paths(service_dict, working_dir=working_dir) if 'build' in service_dict: service_dict['build'] = resolve_build_path(service_dict['build'], working_dir=working_dir) @@ -421,17 +428,31 @@ def env_vars_from_file(filename): return env -def resolve_volume_paths(volumes, working_dir=None): +def resolve_volume_paths(service_dict, working_dir=None): if working_dir is None: raise Exception("No working_dir passed to resolve_volume_paths()") - return [resolve_volume_path(v, working_dir) for v in volumes] + return [ + resolve_volume_path(v, working_dir, service_dict['name']) + for v in service_dict['volumes'] + ] -def resolve_volume_path(volume, working_dir): +def resolve_volume_path(volume, working_dir, service_name): container_path, host_path = split_path_mapping(volume) container_path = os.path.expanduser(container_path) + if host_path is not None: + if not any(host_path.startswith(c) for c in PATH_START_CHARS): + log.warn( + 'Warning: the mapping "{0}" in the volumes config for ' + 'service "{1}" is ambiguous. In a future version of Docker, ' + 'it will designate a "named" volume ' + '(see https://github.com/docker/docker/pull/14242). ' + 'To prevent unexpected behaviour, change it to "./{0}"' + .format(volume, service_name) + ) + host_path = os.path.expanduser(host_path) return "%s:%s" % (expand_path(working_dir, host_path), container_path) else: diff --git a/docs/yml.md b/docs/yml.md index 18551bf22f..6ac1ce62af 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -135,11 +135,12 @@ Mount paths as volumes, optionally specifying a path on the host machine volumes: - /var/lib/mysql - - cache/:/tmp/cache + - ./cache:/tmp/cache - ~/configs:/etc/configs/:ro You can mount a relative path on the host, which will expand relative to -the directory of the Compose configuration file being used. +the directory of the Compose configuration file being used. Relative paths +should always begin with `.` or `..`. > Note: No path expansion will be done if you have also specified a > `volume_driver`. diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0046202030..a181e79ea0 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -117,6 +117,51 @@ class InterpolationTest(unittest.TestCase): d = make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') self.assertEqual(d['volumes'], ['/home/user:/container/path']) + @mock.patch.dict(os.environ) + def test_volume_binding_with_local_dir_name_raises_warning(self): + def make_dict(**config): + make_service_dict('foo', config, working_dir='.') + + with mock.patch('compose.config.config.log.warn') as warn: + make_dict(volumes=['/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['/data:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['.:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['..:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['./data:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['../data:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['.profile:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['~:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['~/data:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['~tmp:/container/path']) + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['data:/container/path'], volume_driver='mydriver') + self.assertEqual(0, warn.call_count) + + make_dict(volumes=['data:/container/path']) + self.assertEqual(1, warn.call_count) + warning = warn.call_args[0][0] + self.assertIn('"data:/container/path"', warning) + self.assertIn('"./data:/container/path"', warning) + def test_named_volume_with_driver_does_not_expand(self): d = make_service_dict('foo', { 'volumes': ['namedvolume:/data'], From b4872de2135a41481f3cbfe97c75126b26c11929 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 10 Aug 2015 14:55:33 +0100 Subject: [PATCH 042/337] Allow integer value for ports While it was intended as a positive to be stricter in validation it would in fact break backwards compatibility, which we do not want to be doing. Consider re-visiting this later and include a deprecation warning if we want to be stricter. Signed-off-by: Mazz Mosley --- compose/config/schema.json | 20 ++++++++++++++++---- compose/config/validation.py | 7 +++++++ tests/unit/config_test.py | 7 ++++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 74f5edbbff..24fd53d116 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -75,10 +75,22 @@ "pid": {"type": "string"}, "ports": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true, - "format": "ports" + "oneOf": [ + { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true, + "format": "ports" + }, + { + "type": "string", + "format": "ports" + }, + { + "type": "number", + "format": "ports" + } + ] }, "privileged": {"type": "string"}, diff --git a/compose/config/validation.py b/compose/config/validation.py index 15e0754cf8..3f46632b85 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -84,6 +84,13 @@ def process_errors(errors): required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) else: required.append(error.message) + elif error.validator == 'oneOf': + config_key = error.path[1] + valid_types = [context.validator_value for context in error.context] + valid_type_msg = " or ".join(valid_types) + type_errors.append("Service '{}' configuration key '{}' contains an invalid type, it should be either {}".format( + service_name, config_key, valid_type_msg) + ) elif error.validator == 'type': msg = "a" if error.validator_value == "array": diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 4e982bb49a..b4d2ce82f6 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -75,8 +75,9 @@ class ConfigTest(unittest.TestCase): ) def test_config_invalid_ports_format_validation(self): - with self.assertRaises(ConfigurationError): - for invalid_ports in [{"1": "8000"}, "whatport", "625", "8000:8050"]: + expected_error_msg = "Service 'web' configuration key 'ports' contains an invalid type" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + for invalid_ports in [{"1": "8000"}, False, 0]: config.load( config.ConfigDetails( {'web': {'image': 'busybox', 'ports': invalid_ports}}, @@ -86,7 +87,7 @@ class ConfigTest(unittest.TestCase): ) def test_config_valid_ports_format_validation(self): - valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"]] + valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], "8000", 8000] for ports in valid_ports: config.load( config.ConfigDetails( From ece6a7271259f6d72586eaa066ebb3034e62ff79 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 10 Aug 2015 15:33:47 +0100 Subject: [PATCH 043/337] Clean error.message Unfortunately the way that jsonschema is calling %r on its property and then encoding the complete message means I've had to do this manual way of removing the literal string prefix, u'. eg: key = 'extends' message = "Invalid value for %r" % key error.message = message.encode("utf-8")" results in: "Invalid value for u'extends'" Performing a replace to strip out the extra "u'", does not change the encoding of the string, it is at this point the character u followed by a '. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 9 ++++++--- tests/unit/config_test.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 3f46632b85..7347c0128d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -52,6 +52,9 @@ def process_errors(errors): def _parse_key_from_error_msg(error): return error.message.split("'")[1] + def _clean_error_message(message): + return message.replace("u'", "'") + root_msgs = [] invalid_keys = [] required = [] @@ -68,7 +71,7 @@ def process_errors(errors): msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS) root_msgs.append(msg) else: - root_msgs.append(error.message) + root_msgs.append(_clean_error_message(error.message)) else: # handle service level errors @@ -83,7 +86,7 @@ def process_errors(errors): elif 'image' not in error.instance and 'build' not in error.instance: required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) else: - required.append(error.message) + required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': config_key = error.path[1] valid_types = [context.validator_value for context in error.context] @@ -104,7 +107,7 @@ def process_errors(errors): root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key)) elif error.validator == 'required': config_key = error.path[1] - required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message)) + required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) elif error.validator == 'dependencies': dependency_key = error.validator_value.keys()[0] required_keys = ",".join(error.validator_value[dependency_key]) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index b4d2ce82f6..e023153a33 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -659,7 +659,7 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_missing_service_key(self): - with self.assertRaisesRegexp(ConfigurationError, "u'service' is a required property"): + with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): config.load( config.ConfigDetails( { From e0675b50c0fa0cc737bfd814d45e495e3d7eaba6 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 10 Aug 2015 16:18:21 +0100 Subject: [PATCH 044/337] Retrieve sub property keys The validation message was confusing by displaying only 1 level of property of the service, even if the error was another level down. Eg. if the 'files' property of 'extends' was the incorrect format, it was displaying 'an invalid value for 'extends'', rather than correctly retrieving 'files'. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 14 ++++++++------ tests/unit/config_test.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 7347c0128d..aa2e0fcf4c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -99,12 +99,14 @@ def process_errors(errors): if error.validator_value == "array": msg = "an" - try: - config_key = error.path[1] - type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value)) - except IndexError: - config_key = error.path[0] - root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key)) + # pop the service name off our path + error.path.popleft() + + if len(error.path) > 0: + config_key = " ".join(["'%s'" % k for k in error.path]) + type_errors.append("Service '{}' configuration key {} contains an invalid type, it should be {} {}".format(service_name, config_key, msg, error.validator_value)) + else: + root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(service_name)) elif error.validator == 'required': config_key = error.path[1] required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index e023153a33..0ea375db49 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -124,7 +124,7 @@ class ConfigTest(unittest.TestCase): ) def test_invalid_config_type_should_be_an_array(self): - expected_error_msg = "Service 'foo' has an invalid value for 'links', it should be an array" + expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( @@ -690,6 +690,25 @@ class ExtendsTest(unittest.TestCase): ) ) + def test_extends_validation_sub_property_key(self): + expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'web': { + 'image': 'busybox', + 'extends': { + 'file': 1, + 'service': 'web', + } + }, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + def test_extends_validation_no_file_key_no_filename_set(self): dictionary = {'extends': {'service': 'web'}} From df14a4384d44f6a27063de08e66d25854509f8d3 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 10 Aug 2015 16:57:32 +0100 Subject: [PATCH 045/337] Catch non-unique errors When a schema type is set as unique, we should display the validation error to indicate that non-unique values have been provided for a key. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 10 +++++++++- tests/unit/config_test.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index aa2e0fcf4c..07b542a116 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -59,6 +59,7 @@ def process_errors(errors): invalid_keys = [] required = [] type_errors = [] + other_errors = [] for error in errors: # handle root level errors @@ -115,8 +116,15 @@ def process_errors(errors): required_keys = ",".join(error.validator_value[dependency_key]) required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( dependency_key, service_name, dependency_key, required_keys)) + else: + # pop the service name off our path + error.path.popleft() - return "\n".join(root_msgs + invalid_keys + required + type_errors) + config_key = " ".join(["'%s'" % k for k in error.path]) + err_msg = "Service '{}' configuration key {} value {}".format(service_name, config_key, error.message) + other_errors.append(err_msg) + + return "\n".join(root_msgs + invalid_keys + required + type_errors + other_errors) def validate_against_schema(config): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0ea375db49..f35010c695 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -147,6 +147,19 @@ class ConfigTest(unittest.TestCase): ) ) + def test_invalid_config_not_unique_items(self): + expected_error_msg = "has non-unique elements" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']} + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 68de84a0bfeb60c4660f110cde850ac17ce3672a Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 10 Aug 2015 17:12:37 +0100 Subject: [PATCH 046/337] Clean up error.path handling Tiny bit of refactoring to make it clearer and only pop service_name once. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 07b542a116..946acf149b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -78,6 +78,9 @@ def process_errors(errors): # handle service level errors service_name = error.path[0] + # pop the service name off our path + error.path.popleft() + if error.validator == 'additionalProperties': invalid_config_key = _parse_key_from_error_msg(error) invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) @@ -89,7 +92,7 @@ def process_errors(errors): else: required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': - config_key = error.path[1] + config_key = error.path[0] valid_types = [context.validator_value for context in error.context] valid_type_msg = " or ".join(valid_types) type_errors.append("Service '{}' configuration key '{}' contains an invalid type, it should be either {}".format( @@ -100,16 +103,13 @@ def process_errors(errors): if error.validator_value == "array": msg = "an" - # pop the service name off our path - error.path.popleft() - if len(error.path) > 0: config_key = " ".join(["'%s'" % k for k in error.path]) type_errors.append("Service '{}' configuration key {} contains an invalid type, it should be {} {}".format(service_name, config_key, msg, error.validator_value)) else: root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(service_name)) elif error.validator == 'required': - config_key = error.path[1] + config_key = error.path[0] required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) elif error.validator == 'dependencies': dependency_key = error.validator_value.keys()[0] @@ -117,9 +117,6 @@ def process_errors(errors): required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( dependency_key, service_name, dependency_key, required_keys)) else: - # pop the service name off our path - error.path.popleft() - config_key = " ".join(["'%s'" % k for k in error.path]) err_msg = "Service '{}' configuration key {} value {}".format(service_name, config_key, error.message) other_errors.append(err_msg) From f8efb54c80ed661538f06ecea7c1329925442b3a Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 11 Aug 2015 13:06:32 +0100 Subject: [PATCH 047/337] Handle $ref defined types errors We use $ref in the schema to allow us to specify multiple type, eg command, it can be a string or a list of strings. It required some extra parsing to retrieve a helpful type to display in our error message rather than 'string or string'. Which while correct, is not helpful. We value helpful. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 16 ++++++++++++++-- tests/unit/config_test.py | 13 +++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 946acf149b..36fd03b5f2 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -55,6 +55,16 @@ def process_errors(errors): def _clean_error_message(message): return message.replace("u'", "'") + def _parse_valid_types_from_schema(schema): + """ + Our defined types using $ref in the schema require some extra parsing + retrieve a helpful type for error message display. + """ + if '$ref' in schema: + return schema['$ref'].replace("#/definitions/", "").replace("_", " ") + else: + return str(schema['type']) + root_msgs = [] invalid_keys = [] required = [] @@ -93,9 +103,11 @@ def process_errors(errors): required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': config_key = error.path[0] - valid_types = [context.validator_value for context in error.context] + + valid_types = [_parse_valid_types_from_schema(schema) for schema in error.schema['oneOf']] valid_type_msg = " or ".join(valid_types) - type_errors.append("Service '{}' configuration key '{}' contains an invalid type, it should be either {}".format( + + type_errors.append("Service '{}' configuration key '{}' contains an invalid type, valid types are {}".format( service_name, config_key, valid_type_msg) ) elif error.validator == 'type': diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f35010c695..1948e21826 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -160,6 +160,19 @@ class ConfigTest(unittest.TestCase): ) ) + def test_invalid_list_of_strings_format(self): + expected_error_msg = "'command' contains an invalid type, valid types are string or list of strings" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + { + 'web': {'build': '.', 'command': [1]} + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 982a8456352e2e22ac2297ddf128ddb9fb51d6ed Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 11 Aug 2015 14:17:30 +0100 Subject: [PATCH 048/337] Fix mem_limit and memswap_limit regression Signed-off-by: Aanand Prasad --- compose/service.py | 4 ++++ tests/unit/service_test.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 2cdd6c9b58..ab7d154e6e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -42,6 +42,8 @@ DOCKER_START_KEYS = [ 'net', 'log_driver', 'log_opt', + 'mem_limit', + 'memswap_limit', 'pid', 'privileged', 'restart', @@ -684,6 +686,8 @@ class Service(object): restart_policy=restart, cap_add=cap_add, cap_drop=cap_drop, + mem_limit=options.get('mem_limit'), + memswap_limit=options.get('memswap_limit'), log_config=log_config, extra_hosts=extra_hosts, read_only=read_only, diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0e274a3583..7e5266dd79 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -173,8 +173,8 @@ class ServiceTest(unittest.TestCase): service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'some': 'overrides'}, 1) - self.assertEqual(opts['memswap_limit'], 2000000000) - self.assertEqual(opts['mem_limit'], 1000000000) + self.assertEqual(opts['host_config']['MemorySwap'], 2000000000) + self.assertEqual(opts['host_config']['Memory'], 1000000000) def test_log_opt(self): log_opt = {'address': 'tcp://192.168.0.42:123'} From 810bb702495f1d6d15008ed0cf87956b863a7ad7 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 11 Aug 2015 16:31:56 +0100 Subject: [PATCH 049/337] Include schema in manifest Signed-off-by: Mazz Mosley --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 6c756417e0..7d48d347a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md +include compose/config/schema.json recursive-include contrib/completion * recursive-include tests * global-exclude *.pyc From d454a584da288b5cc4ecc30d85f57a02931dac69 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Tue, 11 Aug 2015 09:38:49 -0700 Subject: [PATCH 050/337] Fixing links after crawl Signed-off-by: Mary Anthony --- docs/completion.md | 2 +- docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 4 ++-- docs/index.md | 4 ++-- docs/install.md | 2 +- docs/production.md | 10 +++++----- docs/rails.md | 2 +- docs/reference/index.md | 2 +- docs/wordpress.md | 2 +- docs/yml.md | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 41ef88e62d..7b8a6733e5 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -64,6 +64,6 @@ Enjoy working with Compose faster and with less typos! - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/django.md b/docs/django.md index 71df4e1168..7e476b3569 100644 --- a/docs/django.md +++ b/docs/django.md @@ -129,7 +129,7 @@ example, run `docker-compose up` and in another terminal run: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/env.md b/docs/env.md index afeb829e72..8ead34f01f 100644 --- a/docs/env.md +++ b/docs/env.md @@ -44,6 +44,6 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose command line completion](completion.md) diff --git a/docs/extends.md b/docs/extends.md index 7a92b771a3..18a072a82d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -78,7 +78,7 @@ For full details on how to use `extends`, refer to the [reference](#reference). ### Example use case In this example, you’ll repurpose the example app from the [quick start -guide](index.md). (If you're not familiar with Compose, it's recommended that +guide](/). (If you're not familiar with Compose, it's recommended that you go through the quick start first.) This example assumes you want to use Compose both to develop an application locally and then deploy it to a production environment. @@ -358,6 +358,6 @@ locally-defined bindings taking precedence: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose command line completion](completion.md) diff --git a/docs/index.md b/docs/index.md index 6d949f88d3..872b015881 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,7 +53,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) @@ -201,7 +201,7 @@ At this point, you have seen the basics of how Compose works. - Next, try the quick start guide for [Django](django.md), [Rails](rails.md), or [Wordpress](wordpress.md). -- See the reference guides for complete details on the [commands](cli.md), the +- See the reference guides for complete details on the [commands](/reference), the [configuration file](yml.md) and [environment variables](env.md). ## Release Notes diff --git a/docs/install.md b/docs/install.md index fa36791927..d71aa0800d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -98,7 +98,7 @@ To uninstall Docker Compose if you installed using `pip`: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/production.md b/docs/production.md index 294f3c4e86..6005113695 100644 --- a/docs/production.md +++ b/docs/production.md @@ -15,8 +15,7 @@ weight=1 While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide can help. The project is actively working towards becoming -production-ready; to learn more about the progress being made, check out the -[roadmap](https://github.com/docker/compose/blob/master/ROADMAP.md) for details +production-ready; to learn more about the progress being made, check out the roadmap for details on how it's coming along and what still needs to be done. When deploying to production, you'll almost certainly want to make changes to @@ -80,8 +79,9 @@ system, exposes the same API as a single Docker host, which means you can use Compose against a Swarm instance and run your apps across multiple hosts. Compose/Swarm integration is still in the experimental stage, and Swarm is still -in beta, but if you'd like to explore and experiment, check out the -[integration guide](https://github.com/docker/compose/blob/master/SWARM.md). +in beta, but if you'd like to explore and experiment, check out the integration +guide. ## Compose documentation @@ -89,7 +89,7 @@ in beta, but if you'd like to explore and experiment, check out the - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/rails.md b/docs/rails.md index 9ce6c4a6f8..b73be90cb5 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -127,7 +127,7 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/reference/index.md b/docs/reference/index.md index 3d3d55d82a..5651e5bf05 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -13,7 +13,7 @@ parent = "smn_compose_ref" The following pages describe the usage information for the [docker-compose](/reference/docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. -* [build](/reference/reference/build.md) +* [build](/reference/build.md) * [help](/reference/help.md) * [kill](/reference/kill.md) * [ps](/reference/ps.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index eda755c178..8440fdbb41 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -117,7 +117,7 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/yml.md b/docs/yml.md index 6ac1ce62af..8e7cf3bbfd 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -408,6 +408,6 @@ dollar sign (`$$`). - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) -- [Command line reference](cli.md) +- [Command line reference](/reference) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) From 192dda4140f592a5db53f44cb5cd8d5b1a3f0ca1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 11 Aug 2015 18:41:21 +0100 Subject: [PATCH 051/337] Bump 1.5.0dev Signed-off-by: Aanand Prasad --- CHANGES.md | 39 +++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 38a5432499..88e725da61 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,45 @@ Change log ========== +1.4.0 (2015-08-04) +------------------ + +- By default, `docker-compose up` now only recreates containers for services whose configuration has changed since they were created. This should result in a dramatic speed-up for many applications. + + The experimental `--x-smart-recreate` flag which introduced this feature in Compose 1.3.0 has been removed, and a `--force-recreate` flag has been added for when you want to recreate everything. + +- Several of Compose's commands - `scale`, `stop`, `kill` and `rm` - now perform actions on multiple containers in parallel, rather than in sequence, which will run much faster on larger applications. + +- You can now specify a custom name for a service's container with `container_name`. Because Docker container names must be unique, this means you can't scale the service beyond one container. + +- You no longer have to specify a `file` option when using `extends` - it will default to the current file. + +- Service names can now contain dots, dashes and underscores. + +- Compose can now read YAML configuration from standard input, rather than from a file, by specifying `-` as the filename. This makes it easier to generate configuration dynamically: + + $ echo 'redis: {"image": "redis"}' | docker-compose --file - up + +- There's a new `docker-compose version` command which prints extended information about Compose's bundled dependencies. + +- `docker-compose.yml` now supports `log_opt` as well as `log_driver`, allowing you to pass extra configuration to a service's logging driver. + +- `docker-compose.yml` now supports `memswap_limit`, similar to `docker run --memory-swap`. + +- When mounting volumes with the `volumes` option, you can now pass in any mode supported by the daemon, not just `:ro` or `:rw`. For example, SELinux users can pass `:z` or `:Z`. + +- You can now specify a custom volume driver with the `volume_driver` option in `docker-compose.yml`, much like `docker run --volume-driver`. + +- A bug has been fixed where Compose would fail to pull images from private registries serving plain (unsecured) HTTP. The `--allow-insecure-ssl` flag, which was previously used to work around this issue, has been deprecated and now has no effect. + +- A bug has been fixed where `docker-compose build` would fail if the build depended on a private Hub image or an image from a private registry. + +- A bug has been fixed where Compose would crash if there were containers which the Docker daemon had not finished removing. + +- Two bugs have been fixed where Compose would sometimes fail with a "Duplicate bind mount" error, or fail to attach volumes to a container, if there was a volume path specified in `docker-compose.yml` with a trailing slash. + +Thanks @mnowster, @dnephin, @ekristen, @funkyfuture, @jeffk and @lukemarsden! + 1.3.3 (2015-07-15) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 0d464ee86a..e3ace98356 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.4.0dev' +__version__ = '1.5.0dev' From 5e2ecff8a15f592b755bb1fffba69992cda450d9 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 12 Aug 2015 15:19:28 +0100 Subject: [PATCH 052/337] Fix ports validation I had misunderstood the valid formats allowed for ports. They must always be in a list. Signed-off-by: Mazz Mosley --- compose/config/schema.json | 30 ++++++++++++++---------------- tests/unit/config_test.py | 4 ++-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 24fd53d116..b615aa2016 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -75,22 +75,20 @@ "pid": {"type": "string"}, "ports": { - "oneOf": [ - { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true, - "format": "ports" - }, - { - "type": "string", - "format": "ports" - }, - { - "type": "number", - "format": "ports" - } - ] + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "format": "ports" + }, + { + "type": "number", + "format": "ports" + } + ] + }, + "uniqueItems": true }, "privileged": {"type": "string"}, diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 44d757d6b4..136a11834c 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -77,7 +77,7 @@ class ConfigTest(unittest.TestCase): def test_config_invalid_ports_format_validation(self): expected_error_msg = "Service 'web' configuration key 'ports' contains an invalid type" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - for invalid_ports in [{"1": "8000"}, False, 0]: + for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: config.load( config.ConfigDetails( {'web': {'image': 'busybox', 'ports': invalid_ports}}, @@ -87,7 +87,7 @@ class ConfigTest(unittest.TestCase): ) def test_config_valid_ports_format_validation(self): - valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], "8000", 8000] + valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] for ports in valid_ports: config.load( config.ConfigDetails( From bcb977425b74fcea8c8b4ff91295b535fc0e58ea Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 12 Aug 2015 15:36:10 +0100 Subject: [PATCH 053/337] Only use overlay driver in CI Signed-off-by: Aanand Prasad --- script/ci | 1 + script/test-versions | 1 + script/wrapdocker | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/script/ci b/script/ci index 2e4ec9197f..b497548781 100755 --- a/script/ci +++ b/script/ci @@ -9,6 +9,7 @@ set -e export DOCKER_VERSIONS=all +export DOCKER_DAEMON_ARGS="--storage-driver=overlay" . script/test-versions >&2 echo "Building Linux binary" diff --git a/script/test-versions b/script/test-versions index 9e81a515d9..ae9620e384 100755 --- a/script/test-versions +++ b/script/test-versions @@ -21,6 +21,7 @@ for version in $DOCKER_VERSIONS; do --volume="/var/lib/docker" \ --volume="${COVERAGE_DIR:-$(pwd)/coverage-html}:/code/coverage-html" \ -e "DOCKER_VERSION=$version" \ + -e "DOCKER_DAEMON_ARGS" \ --entrypoint="script/dind" \ "$TAG" \ script/wrapdocker nosetests --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html "$@" diff --git a/script/wrapdocker b/script/wrapdocker index 3e669b5d7a..ab89f5ed64 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -7,7 +7,9 @@ fi # If a pidfile is still around (for example after a container restart), # delete it so that docker can start. rm -rf /var/run/docker.pid -docker -d --storage-driver="overlay" &>/var/log/docker.log & +docker_command="docker -d $DOCKER_DAEMON_ARGS" +>&2 echo "Starting Docker with: $docker_command" +$docker_command &>/var/log/docker.log & docker_pid=$! >&2 echo "Waiting for Docker to start..." From 4c65891db10250705015407cb914b40d6eaa3378 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 7 Aug 2015 15:04:30 +0100 Subject: [PATCH 054/337] Avoid duplicate warnings if an unset env variable is used multiple times Signed-off-by: Aanand Prasad --- compose/config/interpolation.py | 38 ++++++++++++++++++-------------- tests/unit/config_test.py | 24 ++++++++++++++++++++ tests/unit/interpolation_test.py | 31 +++++++++++++------------- 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index d33e93be49..8ebcc87596 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -10,13 +10,15 @@ log = logging.getLogger(__name__) def interpolate_environment_variables(config): + mapping = BlankDefaultDict(os.environ) + return dict( - (service_name, process_service(service_name, service_dict)) + (service_name, process_service(service_name, service_dict, mapping)) for (service_name, service_dict) in config.items() ) -def process_service(service_name, service_dict): +def process_service(service_name, service_dict, mapping): if not isinstance(service_dict, dict): raise ConfigurationError( 'Service "%s" doesn\'t have any configuration options. ' @@ -25,14 +27,14 @@ def process_service(service_name, service_dict): ) return dict( - (key, interpolate_value(service_name, key, val)) + (key, interpolate_value(service_name, key, val, mapping)) for (key, val) in service_dict.items() ) -def interpolate_value(service_name, config_key, value): +def interpolate_value(service_name, config_key, value, mapping): try: - return recursive_interpolate(value) + return recursive_interpolate(value, mapping) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' @@ -45,39 +47,43 @@ def interpolate_value(service_name, config_key, value): ) -def recursive_interpolate(obj): +def recursive_interpolate(obj, mapping): if isinstance(obj, six.string_types): - return interpolate(obj, os.environ) + return interpolate(obj, mapping) elif isinstance(obj, dict): return dict( - (key, recursive_interpolate(val)) + (key, recursive_interpolate(val, mapping)) for (key, val) in obj.items() ) elif isinstance(obj, list): - return map(recursive_interpolate, obj) + return [recursive_interpolate(val, mapping) for val in obj] else: return obj def interpolate(string, mapping): try: - return Template(string).substitute(BlankDefaultDict(mapping)) + return Template(string).substitute(mapping) except ValueError: raise InvalidInterpolation(string) class BlankDefaultDict(dict): - def __init__(self, mapping): - super(BlankDefaultDict, self).__init__(mapping) + def __init__(self, *args, **kwargs): + super(BlankDefaultDict, self).__init__(*args, **kwargs) + self.missing_keys = [] def __getitem__(self, key): try: return super(BlankDefaultDict, self).__getitem__(key) except KeyError: - log.warn( - "The {} variable is not set. Substituting a blank string." - .format(key) - ) + if key not in self.missing_keys: + log.warn( + "The {} variable is not set. Substituting a blank string." + .format(key) + ) + self.missing_keys.append(key) + return "" diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 44d757d6b4..6995997948 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -198,6 +198,30 @@ class InterpolationTest(unittest.TestCase): } ]) + @mock.patch.dict(os.environ) + def test_unset_variable_produces_warning(self): + os.environ.pop('FOO', None) + os.environ.pop('BAR', None) + config_details = config.ConfigDetails( + config={ + 'web': { + 'image': '${FOO}', + 'command': '${BAR}', + 'entrypoint': '${BAR}', + }, + }, + working_dir='.', + filename=None, + ) + + with mock.patch('compose.config.interpolation.log') as log: + config.load(config_details) + + self.assertEqual(2, log.warn.call_count) + warnings = sorted(args[0][0] for args in log.warn.call_args_list) + self.assertIn('BAR', warnings[0]) + self.assertIn('FOO', warnings[1]) + @mock.patch.dict(os.environ) def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py index 96c6f9b33a..fb95422b0e 100644 --- a/tests/unit/interpolation_test.py +++ b/tests/unit/interpolation_test.py @@ -1,31 +1,32 @@ import unittest from compose.config.interpolation import interpolate, InvalidInterpolation +from compose.config.interpolation import BlankDefaultDict as bddict class InterpolationTest(unittest.TestCase): def test_valid_interpolations(self): - self.assertEqual(interpolate('$foo', dict(foo='hi')), 'hi') - self.assertEqual(interpolate('${foo}', dict(foo='hi')), 'hi') + self.assertEqual(interpolate('$foo', bddict(foo='hi')), 'hi') + self.assertEqual(interpolate('${foo}', bddict(foo='hi')), 'hi') - self.assertEqual(interpolate('${subject} love you', dict(subject='i')), 'i love you') - self.assertEqual(interpolate('i ${verb} you', dict(verb='love')), 'i love you') - self.assertEqual(interpolate('i love ${object}', dict(object='you')), 'i love you') + self.assertEqual(interpolate('${subject} love you', bddict(subject='i')), 'i love you') + self.assertEqual(interpolate('i ${verb} you', bddict(verb='love')), 'i love you') + self.assertEqual(interpolate('i love ${object}', bddict(object='you')), 'i love you') def test_empty_value(self): - self.assertEqual(interpolate('${foo}', dict(foo='')), '') + self.assertEqual(interpolate('${foo}', bddict(foo='')), '') def test_unset_value(self): - self.assertEqual(interpolate('${foo}', dict()), '') + self.assertEqual(interpolate('${foo}', bddict()), '') def test_escaped_interpolation(self): - self.assertEqual(interpolate('$${foo}', dict(foo='hi')), '${foo}') + self.assertEqual(interpolate('$${foo}', bddict(foo='hi')), '${foo}') def test_invalid_strings(self): - self.assertRaises(InvalidInterpolation, lambda: interpolate('${', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('$}', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${}', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${ }', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${ foo}', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo }', dict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo!}', dict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('$}', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${}', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${ }', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${ foo}', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo }', bddict())) + self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo!}', bddict())) From f1eef7b416b1ebe4e0e26e6179a5df02343348f1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 12 Aug 2015 16:31:37 +0100 Subject: [PATCH 055/337] Fill out release process documentation Signed-off-by: Aanand Prasad --- RELEASE_PROCESS.md | 147 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 121 insertions(+), 26 deletions(-) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 86522faaf3..e81a55ec69 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -1,36 +1,131 @@ -# Building a Compose release +Building a Compose release +========================== -## Building binaries +## To get started with a new release -`script/build-linux` builds the Linux binary inside a Docker container: +1. Create a `bump-$VERSION` branch off master: - $ script/build-linux + git checkout -b bump-$VERSION master -`script/build-osx` builds the Mac OS X binary inside a virtualenv: +2. Merge in the `release` branch on the upstream repo, discarding its tree entirely: - $ script/build-osx + git fetch origin + git merge --strategy=ours origin/release -For official releases, you should build inside a Mountain Lion VM for proper -compatibility. Run the this script first to prepare the environment before -building - it will use Homebrew to make sure Python is installed and -up-to-date. +3. Update the version in `docs/install.md` and `compose/__init__.py`. - $ script/prepare-osx + If the next release will be an RC, append `rcN`, e.g. `1.4.0rc1`. -## Release process +4. Write release notes in `CHANGES.md`. -1. Open pull request that: - - Updates the version in `compose/__init__.py` - - Updates the binary URL in `docs/install.md` - - Adds release notes to `CHANGES.md` -2. Create unpublished GitHub release with release notes -3. Build Linux version on any Docker host with `script/build-linux` and attach - to release -4. Build OS X version on Mountain Lion with `script/build-osx` and attach to - release as `docker-compose-Darwin-x86_64` and `docker-compose-Linux-x86_64`. -5. Publish GitHub release, creating tag -6. Update website with `script/deploy-docs` -7. Upload PyPi package +5. Add a bump commit: - $ git checkout $VERSION - $ python setup.py sdist upload + git commit -am "Bump $VERSION" + +6. Push the bump branch to your fork: + + git push --set-upstream $USERNAME bump-$VERSION + +7. Open a PR from the bump branch against the `release` branch on the upstream repo, **not** against master. + +## When a PR is merged into master that we want in the release + +1. Check out the bump branch: + + git checkout bump-$VERSION + +2. Cherry-pick the merge commit, fixing any conflicts if necessary: + + git cherry-pick -xm1 $MERGE_COMMIT_HASH + +3. Add a signoff (it’s missing from merge commits): + + git commit --amend --signoff + +4. Move the bump commit back to the tip of the branch: + + git rebase --interactive $PARENT_OF_BUMP_COMMIT + +5. Force-push the bump branch to your fork: + + git push --force $USERNAME bump-$VERSION + +## To release a version (whether RC or stable) + +1. Check that CI is passing on the bump PR. + +2. Check out the bump branch: + + git checkout bump-$VERSION + +3. Build the Linux binary: + + script/build-linux + +4. Build the Mac binary in a Mountain Lion VM: + + script/prepare-osx + script/build-osx + +5. Test the binaries and/or get some other people to test them. + +6. Create a tag: + + TAG=$VERSION # or $VERSION-rcN, if it's an RC + git tag $TAG + +7. Push the tag to the upstream repo: + + git push git@github.com:docker/compose.git $TAG + +8. Create a release from the tag on GitHub. + +9. Paste in installation instructions and release notes. + +10. Attach the binaries. + +11. Don’t publish it just yet! + +12. Upload the latest version to PyPi: + + python setup.py sdist upload + +13. Check that the pip package installs and runs (best done in a virtualenv): + + pip install -U docker-compose==$TAG + docker-compose version + +14. Publish the release on GitHub. + +15. Check that both binaries download (following the install instructions) and run. + +16. Email maintainers@dockerproject.org and engineering@docker.com about the new release. + +## If it’s a stable release (not an RC) + +1. Merge the bump PR. + +2. Make sure `origin/release` is updated locally: + + git fetch origin + +3. Update the `docs` branch on the upstream repo: + + git push git@github.com:docker/compose.git origin/release:docs + +4. Let the docs team know that it’s been updated so they can publish it. + +5. Close the release’s milestone. + +## If it’s a minor release (1.x.0), rather than a patch release (1.x.y) + +1. Open a PR against `master` to: + + - update `CHANGELOG.md` to bring it in line with `release` + - bump the version in `compose/__init__.py` to the *next* minor version number with `dev` appended. For example, if you just released `1.4.0`, update it to `1.5.0dev`. + +2. Get the PR merged. + +## Finally + +1. Celebrate, however you’d like. From 440099754d9eb45b953c3db3baecf8219e2a8e1c Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 12 Aug 2015 17:29:33 +0100 Subject: [PATCH 056/337] memory values can be strings or numbers Signed-off-by: Mazz Mosley --- compose/config/schema.json | 14 ++++++++++++-- tests/unit/config_test.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index b615aa2016..073a0da65a 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -68,8 +68,18 @@ }, "mac_address": {"type": "string"}, - "mem_limit": {"type": "number"}, - "memswap_limit": {"type": "number"}, + "mem_limit": { + "oneOf": [ + {"type": "number"}, + {"type": "string"} + ] + }, + "memswap_limit": { + "oneOf": [ + {"type": "number"}, + {"type": "string"} + ] + }, "name": {"type": "string"}, "net": {"type": "string"}, "pid": {"type": "string"}, diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index c8a2d0db14..861d36bdac 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -533,6 +533,16 @@ class MemoryOptionsTest(unittest.TestCase): ) self.assertEqual(service_dict[0]['memswap_limit'], 2000000) + def test_memswap_can_be_a_string(self): + service_dict = config.load( + config.ConfigDetails( + {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}}, + 'tests/fixtures/extends', + 'common.yml' + ) + ) + self.assertEqual(service_dict[0]['memswap_limit'], "512M") + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): From ff87ceabbd4dcc7f0876f4bef03c4587327fa36b Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Fri, 7 Aug 2015 13:42:37 +0100 Subject: [PATCH 057/337] Allow manual port mapping when using "run" command. Fixes #1709 Signed-off-by: Karol Duleba --- compose/cli/main.py | 12 +++++++- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + docs/reference/run.md | 5 ++++ tests/integration/cli_test.py | 38 ++++++++++++++++++++++++++ tests/unit/cli_test.py | 31 +++++++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 56f6c05052..6c2a8edb61 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -282,7 +282,7 @@ class TopLevelCommand(Command): running. If you do not want to start linked services, use `docker-compose run --no-deps SERVICE COMMAND [ARGS...]`. - Usage: run [options] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] + Usage: run [options] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] Options: --allow-insecure-ssl Deprecated - no effect. @@ -293,6 +293,7 @@ class TopLevelCommand(Command): -u, --user="" Run as specified username or uid --no-deps Don't start linked services. --rm Remove container after run. Ignored in detached mode. + -p, --publish=[] Publish a container's port(s) to the host --service-ports Run command with the service's ports enabled and mapped to the host. -T Disable pseudo-tty allocation. By default `docker-compose run` @@ -344,6 +345,15 @@ class TopLevelCommand(Command): if not options['--service-ports']: container_options['ports'] = [] + if options['--publish']: + container_options['ports'] = options.get('--publish') + + if options['--publish'] and options['--service-ports']: + raise UserError( + 'Service port mapping and manual port mapping ' + 'can not be used togather' + ) + try: container = service.create_container( quiet=True, diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e7d8cb3f8e..128428d9a2 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -248,7 +248,7 @@ _docker-compose_run() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --no-deps --rm --service-ports -T --user -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --no-deps --rm --service-ports --publish -p -T --user -u" -- "$cur" ) ) ;; *) __docker-compose_services_all diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 9af21a98b3..9ac7e7560f 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -221,6 +221,7 @@ __docker-compose_subcommand () { '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ "--no-deps[Don't start linked services.]" \ '--rm[Remove container after run. Ignored in detached mode.]' \ + "--publish[Run command with manually mapped container's port(s) to the host.]" \ "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ '(-):services:__docker-compose_services' \ diff --git a/docs/reference/run.md b/docs/reference/run.md index 5ea9a61bec..93ae0212b2 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -22,6 +22,7 @@ Options: -u, --user="" Run as specified username or uid --no-deps Don't start linked services. --rm Remove container after run. Ignored in detached mode. +-p, --publish=[] Publish a container's port(s) to the host --service-ports Run command with the service's ports enabled and mapped to the host. -T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. ``` @@ -38,6 +39,10 @@ The second difference is the `docker-compose run` command does not create any of $ docker-compose run --service-ports web python manage.py shell +Alternatively manual port mapping can be specified. Same as when running Docker's `run` command - using `--publish` or `-p` options: + + $ docker-compose run --publish 8080:80 -p 2022:22 -p 127.0.0.1:2021:21 web python manage.py shell + If you start a service configured with links, the `run` command first checks to see if the linked service is running and starts the service if it is stopped. Once all the linked services are running, the `run` executes the command you passed it. So, for example, you could run: $ docker-compose run db psql -h db -U docker diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 0e86c2792f..ce497c8228 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -343,6 +343,44 @@ class CLITestCase(DockerClientTestCase): self.assertIn("0.0.0.0", port_random) self.assertEqual(port_assigned, "0.0.0.0:49152") + @patch('dockerpty.start') + def test_run_service_with_explicitly_maped_ports(self, __): + + # create one off container + self.command.base_dir = 'tests/fixtures/ports-composefile' + self.command.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple'], None) + container = self.project.get_service('simple').containers(one_off=True)[0] + + # get port information + port_short = container.get_local_port(3000) + port_full = container.get_local_port(3001) + + # close all one off containers we just created + container.stop() + + # check the ports + self.assertEqual(port_short, "0.0.0.0:30000") + self.assertEqual(port_full, "0.0.0.0:30001") + + @patch('dockerpty.start') + def test_run_service_with_explicitly_maped_ip_ports(self, __): + + # create one off container + self.command.base_dir = 'tests/fixtures/ports-composefile' + self.command.dispatch(['run', '-d', '-p', '127.0.0.1:30000:3000', '--publish', '127.0.0.1:30001:3001', 'simple'], None) + container = self.project.get_service('simple').containers(one_off=True)[0] + + # get port information + port_short = container.get_local_port(3000) + port_full = container.get_local_port(3001) + + # close all one off containers we just created + container.stop() + + # check the ports + self.assertEqual(port_short, "127.0.0.1:30000") + self.assertEqual(port_full, "127.0.0.1:30001") + def test_rm(self): service = self.project.get_service('simple') service.create_container() diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 3f50003292..e11f6f14af 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -7,6 +7,7 @@ import docker import mock from compose.cli.docopt_command import NoSuchCommand +from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.service import Service @@ -108,6 +109,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--publish': [], '--rm': None, }) @@ -136,6 +138,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--publish': [], '--rm': None, }) _, _, call_kwargs = mock_client.create_container.mock_calls[0] @@ -160,7 +163,35 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--publish': [], '--rm': True, }) _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertFalse('RestartPolicy' in call_kwargs['host_config']) + + def test_command_manula_and_service_ports_together(self): + command = TopLevelCommand() + mock_client = mock.create_autospec(docker.Client) + mock_project = mock.Mock(client=mock_client) + mock_project.get_service.return_value = Service( + 'service', + client=mock_client, + restart='always', + image='someimage', + ) + + with self.assertRaises(UserError): + command.run(mock_project, { + 'SERVICE': 'service', + 'COMMAND': None, + '-e': [], + '--user': None, + '--no-deps': None, + '--allow-insecure-ssl': None, + '-d': True, + '-T': None, + '--entrypoint': None, + '--service-ports': True, + '--publish': ['80:80'], + '--rm': None, + }) From 2e7f08c2ef28c375329ab32952b30fe116dadeed Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Mon, 10 Aug 2015 23:16:55 +0100 Subject: [PATCH 058/337] Raise configuration error when trying to extend service that does not exist. Fixes #1826 Signed-off-by: Karol Duleba --- compose/config/config.py | 10 +++++++++- tests/fixtures/extends/nonexistent-service.yml | 4 ++++ tests/unit/config_test.py | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/extends/nonexistent-service.yml diff --git a/compose/config/config.py b/compose/config/config.py index b5646a479c..44c401d4b1 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -186,8 +186,16 @@ class ServiceLoader(object): already_seen=other_already_seen, ) + base_service = extends_options['service'] other_config = load_yaml(other_config_path) - other_service_dict = other_config[extends_options['service']] + + if base_service not in other_config: + msg = ( + "Cannot extend service '%s' in %s: Service not found" + ) % (base_service, other_config_path) + raise ConfigurationError(msg) + + other_service_dict = other_config[base_service] other_loader.detect_cycle(extends_options['service']) other_service_dict = other_loader.make_service_dict( service_dict['name'], diff --git a/tests/fixtures/extends/nonexistent-service.yml b/tests/fixtures/extends/nonexistent-service.yml new file mode 100644 index 0000000000..e9e17f1bdc --- /dev/null +++ b/tests/fixtures/extends/nonexistent-service.yml @@ -0,0 +1,4 @@ +web: + image: busybox + extends: + service: foo diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 861d36bdac..3e3e9e34a6 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -917,6 +917,11 @@ class ExtendsTest(unittest.TestCase): }, ]) + def test_load_throws_error_when_base_service_does_not_exist(self): + err_msg = r'''Cannot extend service 'foo' in .*: Service not found''' + with self.assertRaisesRegexp(ConfigurationError, err_msg): + load_from_filename('tests/fixtures/extends/nonexistent-service.yml') + class BuildPathTest(unittest.TestCase): def setUp(self): From 67995ab9e37cda9c60fd40f96cb77d41bf21de0e Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 12 Aug 2015 17:09:48 +0100 Subject: [PATCH 059/337] Pre-process validation steps In order to validate a service name that has been specified as an integer we need to run that as a pre-process validation step *before* we pass the config to be validated against the schema. It is not possible to validate it *in* the schema, it causes a type error. Even though a number is a valid service name, it must be a cast as a string within the yaml to avoid type error. Taken this opportunity to move the code design in a direction towards: 1. pre-process 2. validate 3. construct Signed-off-by: Mazz Mosley --- compose/config/config.py | 27 +++++++++++++++++++-------- compose/config/validation.py | 24 ++++++++++++++++++++++++ tests/unit/config_test.py | 11 +++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index b5646a479c..239bbf5e00 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -13,7 +13,11 @@ from .errors import ( CircularReference, ComposeFileNotFound, ) -from .validation import validate_against_schema +from .validation import ( + validate_against_schema, + validate_service_names, + validate_top_level_object +) DOCKER_CONFIG_KEYS = [ @@ -122,19 +126,26 @@ def get_config_path(base_dir): return os.path.join(path, winner) +@validate_top_level_object +@validate_service_names +def pre_process_config(config): + """ + Pre validation checks and processing of the config file to interpolate env + vars returning a config dict ready to be tested against the schema. + """ + config = interpolate_environment_variables(config) + return config + + def load(config_details): config, working_dir, filename = config_details - if not isinstance(config, dict): - raise ConfigurationError( - "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - ) - config = interpolate_environment_variables(config) - validate_against_schema(config) + processed_config = pre_process_config(config) + validate_against_schema(processed_config) service_dicts = [] - for service_name, service_dict in list(config.items()): + for service_name, service_dict in list(processed_config.items()): loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) validate_paths(service_dict) diff --git a/compose/config/validation.py b/compose/config/validation.py index 36fd03b5f2..26f3ca8ec7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,3 +1,4 @@ +from functools import wraps import os from docker.utils.ports import split_port @@ -36,6 +37,29 @@ def format_ports(instance): return True +def validate_service_names(func): + @wraps(func) + def func_wrapper(config): + for service_name in config.keys(): + if type(service_name) is int: + raise ConfigurationError( + "Service name: {} needs to be a string, eg '{}'".format(service_name, service_name) + ) + return func(config) + return func_wrapper + + +def validate_top_level_object(func): + @wraps(func) + def func_wrapper(config): + if not isinstance(config, dict): + raise ConfigurationError( + "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." + ) + return func(config) + return func_wrapper + + def get_unsupported_config_msg(service_name, error_key): msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) if error_key in DOCKER_CONFIG_HINTS: diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index c8a2d0db14..553e85beb8 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -64,6 +64,17 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_integer_service_name_raise_validation_error(self): + expected_error_msg = "Service name: 1 needs to be a string, eg '1'" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {1: {'image': 'busybox'}}, + 'working_dir', + 'filename.yml' + ) + ) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From 530d20db6d7929b3d73fb08956c3b4f29520d1bd Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 14 Aug 2015 11:15:22 +0100 Subject: [PATCH 060/337] Fix volume path warning Signed-off-by: Aanand Prasad --- compose/config/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index b5646a479c..e5b80c0150 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -399,12 +399,12 @@ def resolve_volume_path(volume, working_dir, service_name): if host_path is not None: if not any(host_path.startswith(c) for c in PATH_START_CHARS): log.warn( - 'Warning: the mapping "{0}" in the volumes config for ' - 'service "{1}" is ambiguous. In a future version of Docker, ' + 'Warning: the mapping "{0}:{1}" in the volumes config for ' + 'service "{2}" is ambiguous. In a future version of Docker, ' 'it will designate a "named" volume ' '(see https://github.com/docker/docker/pull/14242). ' - 'To prevent unexpected behaviour, change it to "./{0}"' - .format(volume, service_name) + 'To prevent unexpected behaviour, change it to "./{0}:{1}"' + .format(host_path, container_path, service_name) ) host_path = os.path.expanduser(host_path) From 478054af4775d6f78fcca4b479f91bf569ff04f7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 11 Aug 2015 15:01:11 +0100 Subject: [PATCH 061/337] Rename CHANGES.md to CHANGELOG.md To align with the docker/docker repo. Signed-off-by: Aanand Prasad --- CHANGELOG.md | 414 ++++++++++++++++++++++++++++++++++++++++++++++++++ CHANGES.md | 415 +-------------------------------------------------- 2 files changed, 415 insertions(+), 414 deletions(-) create mode 100644 CHANGELOG.md mode change 100644 => 120000 CHANGES.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..88e725da61 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,414 @@ +Change log +========== + +1.4.0 (2015-08-04) +------------------ + +- By default, `docker-compose up` now only recreates containers for services whose configuration has changed since they were created. This should result in a dramatic speed-up for many applications. + + The experimental `--x-smart-recreate` flag which introduced this feature in Compose 1.3.0 has been removed, and a `--force-recreate` flag has been added for when you want to recreate everything. + +- Several of Compose's commands - `scale`, `stop`, `kill` and `rm` - now perform actions on multiple containers in parallel, rather than in sequence, which will run much faster on larger applications. + +- You can now specify a custom name for a service's container with `container_name`. Because Docker container names must be unique, this means you can't scale the service beyond one container. + +- You no longer have to specify a `file` option when using `extends` - it will default to the current file. + +- Service names can now contain dots, dashes and underscores. + +- Compose can now read YAML configuration from standard input, rather than from a file, by specifying `-` as the filename. This makes it easier to generate configuration dynamically: + + $ echo 'redis: {"image": "redis"}' | docker-compose --file - up + +- There's a new `docker-compose version` command which prints extended information about Compose's bundled dependencies. + +- `docker-compose.yml` now supports `log_opt` as well as `log_driver`, allowing you to pass extra configuration to a service's logging driver. + +- `docker-compose.yml` now supports `memswap_limit`, similar to `docker run --memory-swap`. + +- When mounting volumes with the `volumes` option, you can now pass in any mode supported by the daemon, not just `:ro` or `:rw`. For example, SELinux users can pass `:z` or `:Z`. + +- You can now specify a custom volume driver with the `volume_driver` option in `docker-compose.yml`, much like `docker run --volume-driver`. + +- A bug has been fixed where Compose would fail to pull images from private registries serving plain (unsecured) HTTP. The `--allow-insecure-ssl` flag, which was previously used to work around this issue, has been deprecated and now has no effect. + +- A bug has been fixed where `docker-compose build` would fail if the build depended on a private Hub image or an image from a private registry. + +- A bug has been fixed where Compose would crash if there were containers which the Docker daemon had not finished removing. + +- Two bugs have been fixed where Compose would sometimes fail with a "Duplicate bind mount" error, or fail to attach volumes to a container, if there was a volume path specified in `docker-compose.yml` with a trailing slash. + +Thanks @mnowster, @dnephin, @ekristen, @funkyfuture, @jeffk and @lukemarsden! + +1.3.3 (2015-07-15) +------------------ + +Two regressions have been fixed: + +- When stopping containers gracefully, Compose was setting the timeout to 0, effectively forcing a SIGKILL every time. +- Compose would sometimes crash depending on the formatting of container data returned from the Docker API. + +1.3.2 (2015-07-14) +------------------ + +The following bugs have been fixed: + +- When there were one-off containers created by running `docker-compose run` on an older version of Compose, `docker-compose run` would fail with a name collision. Compose now shows an error if you have leftover containers of this type lying around, and tells you how to remove them. +- Compose was not reading Docker authentication config files created in the new location, `~/docker/config.json`, and authentication against private registries would therefore fail. +- When a container had a pseudo-TTY attached, its output in `docker-compose up` would be truncated. +- `docker-compose up --x-smart-recreate` would sometimes fail when an image tag was updated. +- `docker-compose up` would sometimes create two containers with the same numeric suffix. +- `docker-compose rm` and `docker-compose ps` would sometimes list services that aren't part of the current project (though no containers were erroneously removed). +- Some `docker-compose` commands would not show an error if invalid service names were passed in. + +Thanks @dano, @josephpage, @kevinsimper, @lieryan, @phemmer, @soulrebel and @sschepens! + +1.3.1 (2015-06-21) +------------------ + +The following bugs have been fixed: + +- `docker-compose build` would always attempt to pull the base image before building. +- `docker-compose help migrate-to-labels` failed with an error. +- If no network mode was specified, Compose would set it to "bridge", rather than allowing the Docker daemon to use its configured default network mode. + +1.3.0 (2015-06-18) +------------------ + +Firstly, two important notes: + +- **This release contains breaking changes, and you will need to either remove or migrate your existing containers before running your app** - see the [upgrading section of the install docs](https://github.com/docker/compose/blob/1.3.0rc1/docs/install.md#upgrading) for details. + +- Compose now requires Docker 1.6.0 or later. + +We've done a lot of work in this release to remove hacks and make Compose more stable: + +- Compose now uses container labels, rather than names, to keep track of containers. This makes Compose both faster and easier to integrate with your own tools. + +- Compose no longer uses "intermediate containers" when recreating containers for a service. This makes `docker-compose up` less complex and more resilient to failure. + +There are some new features: + +- `docker-compose up` has an **experimental** new behaviour: it will only recreate containers for services whose configuration has changed in `docker-compose.yml`. This will eventually become the default, but for now you can take it for a spin: + + $ docker-compose up --x-smart-recreate + +- When invoked in a subdirectory of a project, `docker-compose` will now climb up through parent directories until it finds a `docker-compose.yml`. + +Several new configuration keys have been added to `docker-compose.yml`: + +- `dockerfile`, like `docker build --file`, lets you specify an alternate Dockerfile to use with `build`. +- `labels`, like `docker run --labels`, lets you add custom metadata to containers. +- `extra_hosts`, like `docker run --add-host`, lets you add entries to a container's `/etc/hosts` file. +- `pid: host`, like `docker run --pid=host`, lets you reuse the same PID namespace as the host machine. +- `cpuset`, like `docker run --cpuset-cpus`, lets you specify which CPUs to allow execution in. +- `read_only`, like `docker run --read-only`, lets you mount a container's filesystem as read-only. +- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/reference/run/#security-configuration). +- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/reference/run/#logging-drivers-log-driver). + +Many bugs have been fixed, including the following: + +- The output of `docker-compose run` was sometimes truncated, especially when running under Jenkins. +- A service's volumes would sometimes not update after volume configuration was changed in `docker-compose.yml`. +- Authenticating against third-party registries would sometimes fail. +- `docker-compose run --rm` would fail to remove the container if the service had a `restart` policy in place. +- `docker-compose scale` would refuse to scale a service beyond 1 container if it exposed a specific port number on the host. +- Compose would refuse to create multiple volume entries with the same host path. + +Thanks @ahromis, @albers, @aleksandr-vin, @antoineco, @ccverak, @chernjie, @dnephin, @edmorley, @fordhurley, @josephpage, @KyleJamesWalker, @lsowen, @mchasal, @noironetworks, @sdake, @sdurrheimer, @sherter, @stephenlawrence, @thaJeztah, @thieman, @turtlemonvh, @twhiteman, @vdemeester, @xuxinkun and @zwily! + +1.2.0 (2015-04-16) +------------------ + +- `docker-compose.yml` now supports an `extends` option, which enables a service to inherit configuration from another service in another configuration file. This is really good for sharing common configuration between apps, or for configuring the same app for different environments. Here's the [documentation](https://github.com/docker/compose/blob/master/docs/yml.md#extends). + +- When using Compose with a Swarm cluster, containers that depend on one another will be co-scheduled on the same node. This means that most Compose apps will now work out of the box, as long as they don't use `build`. + +- Repeated invocations of `docker-compose up` when using Compose with a Swarm cluster now work reliably. + +- Directories passed to `build`, filenames passed to `env_file` and volume host paths passed to `volumes` are now treated as relative to the *directory of the configuration file*, not the directory that `docker-compose` is being run in. In the majority of cases, those are the same, but if you use the `-f|--file` argument to specify a configuration file in another directory, **this is a breaking change**. + +- A service can now share another service's network namespace with `net: container:`. + +- `volumes_from` and `net: container:` entries are taken into account when resolving dependencies, so `docker-compose up ` will correctly start all dependencies of ``. + +- `docker-compose run` now accepts a `--user` argument to specify a user to run the command as, just like `docker run`. + +- The `up`, `stop` and `restart` commands now accept a `--timeout` (or `-t`) argument to specify how long to wait when attempting to gracefully stop containers, just like `docker stop`. + +- `docker-compose rm` now accepts `-f` as a shorthand for `--force`, just like `docker rm`. + +Thanks, @abesto, @albers, @alunduil, @dnephin, @funkyfuture, @gilclark, @IanVS, @KingsleyKelly, @knutwalker, @thaJeztah and @vmalloc! + +1.1.0 (2015-02-25) +------------------ + +Fig has been renamed to Docker Compose, or just Compose for short. This has several implications for you: + +- The command you type is now `docker-compose`, not `fig`. +- You should rename your fig.yml to docker-compose.yml. +- If you’re installing via PyPi, the package is now `docker-compose`, so install it with `pip install docker-compose`. + +Besides that, there’s a lot of new stuff in this release: + +- We’ve made a few small changes to ensure that Compose will work with Swarm, Docker’s new clustering tool (https://github.com/docker/swarm). Eventually you'll be able to point Compose at a Swarm cluster instead of a standalone Docker host and it’ll run your containers on the cluster with no extra work from you. As Swarm is still developing, integration is rough and lots of Compose features don't work yet. + +- `docker-compose run` now has a `--service-ports` flag for exposing ports on the given service. This is useful for e.g. running your webapp with an interactive debugger. + +- You can now link to containers outside your app with the `external_links` option in docker-compose.yml. + +- You can now prevent `docker-compose up` from automatically building images with the `--no-build` option. This will make fewer API calls and run faster. + +- If you don’t specify a tag when using the `image` key, Compose will default to the `latest` tag, rather than pulling all tags. + +- `docker-compose kill` now supports the `-s` flag, allowing you to specify the exact signal you want to send to a service’s containers. + +- docker-compose.yml now has an `env_file` key, analogous to `docker run --env-file`, letting you specify multiple environment variables in a separate file. This is great if you have a lot of them, or if you want to keep sensitive information out of version control. + +- docker-compose.yml now supports the `dns_search`, `cap_add`, `cap_drop`, `cpu_shares` and `restart` options, analogous to `docker run`’s `--dns-search`, `--cap-add`, `--cap-drop`, `--cpu-shares` and `--restart` options. + +- Compose now ships with Bash tab completion - see the installation and usage docs at https://github.com/docker/compose/blob/1.1.0/docs/completion.md + +- A number of bugs have been fixed - see the milestone for details: https://github.com/docker/compose/issues?q=milestone%3A1.1.0+ + +Thanks @dnephin, @squebe, @jbalonso, @raulcd, @benlangfield, @albers, @ggtools, @bersace, @dtenenba, @petercv, @drewkett, @TFenby, @paulRbr, @Aigeruth and @salehe! + +1.0.1 (2014-11-04) +------------------ + + - Added an `--allow-insecure-ssl` option to allow `fig up`, `fig run` and `fig pull` to pull from insecure registries. + - Fixed `fig run` not showing output in Jenkins. + - Fixed a bug where Fig couldn't build Dockerfiles with ADD statements pointing at URLs. + +1.0.0 (2014-10-16) +------------------ + +The highlights: + + - [Fig has joined Docker.](https://www.orchardup.com/blog/orchard-is-joining-docker) Fig will continue to be maintained, but we'll also be incorporating the best bits of Fig into Docker itself. + + This means the GitHub repository has moved to [https://github.com/docker/fig](https://github.com/docker/fig) and our IRC channel is now #docker-fig on Freenode. + + - Fig can be used with the [official Docker OS X installer](https://docs.docker.com/installation/mac/). Boot2Docker will mount the home directory from your host machine so volumes work as expected. + + - Fig supports Docker 1.3. + + - It is now possible to connect to the Docker daemon using TLS by using the `DOCKER_CERT_PATH` and `DOCKER_TLS_VERIFY` environment variables. + + - There is a new `fig port` command which outputs the host port binding of a service, in a similar way to `docker port`. + + - There is a new `fig pull` command which pulls the latest images for a service. + + - There is a new `fig restart` command which restarts a service's containers. + + - Fig creates multiple containers in service by appending a number to the service name (e.g. `db_1`, `db_2`, etc). As a convenience, Fig will now give the first container an alias of the service name (e.g. `db`). + + This link alias is also a valid hostname and added to `/etc/hosts` so you can connect to linked services using their hostname. For example, instead of resolving the environment variables `DB_PORT_5432_TCP_ADDR` and `DB_PORT_5432_TCP_PORT`, you could just use the hostname `db` and port `5432` directly. + + - Volume definitions now support `ro` mode, expanding `~` and expanding environment variables. + + - `.dockerignore` is supported when building. + + - The project name can be set with the `FIG_PROJECT_NAME` environment variable. + + - The `--env` and `--entrypoint` options have been added to `fig run`. + + - The Fig binary for Linux is now linked against an older version of glibc so it works on CentOS 6 and Debian Wheezy. + +Other things: + + - `fig ps` now works on Jenkins and makes fewer API calls to the Docker daemon. + - `--verbose` displays more useful debugging output. + - When starting a service where `volumes_from` points to a service without any containers running, that service will now be started. + - Lots of docs improvements. Notably, environment variables are documented and official repositories are used throughout. + +Thanks @dnephin, @d11wtq, @marksteve, @rubbish, @jbalonso, @timfreund, @alunduil, @mieciu, @shuron, @moss, @suzaku and @chmouel! Whew. + +0.5.2 (2014-07-28) +------------------ + + - Added a `--no-cache` option to `fig build`, which bypasses the cache just like `docker build --no-cache`. + - Fixed the `dns:` fig.yml option, which was causing fig to error out. + - Fixed a bug where fig couldn't start under Python 2.6. + - Fixed a log-streaming bug that occasionally caused fig to exit. + +Thanks @dnephin and @marksteve! + + +0.5.1 (2014-07-11) +------------------ + + - If a service has a command defined, `fig run [service]` with no further arguments will run it. + - The project name now defaults to the directory containing fig.yml, not the current working directory (if they're different) + - `volumes_from` now works properly with containers as well as services + - Fixed a race condition when recreating containers in `fig up` + +Thanks @ryanbrainard and @d11wtq! + + +0.5.0 (2014-07-11) +------------------ + + - Fig now starts links when you run `fig run` or `fig up`. + + For example, if you have a `web` service which depends on a `db` service, `fig run web ...` will start the `db` service. + + - Environment variables can now be resolved from the environment that Fig is running in. Just specify it as a blank variable in your `fig.yml` and, if set, it'll be resolved: + ``` + environment: + RACK_ENV: development + SESSION_SECRET: + ``` + + - `volumes_from` is now supported in `fig.yml`. All of the volumes from the specified services and containers will be mounted: + + ``` + volumes_from: + - service_name + - container_name + ``` + + - A host address can now be specified in `ports`: + + ``` + ports: + - "0.0.0.0:8000:8000" + - "127.0.0.1:8001:8001" + ``` + + - The `net` and `workdir` options are now supported in `fig.yml`. + - The `hostname` option now works in the same way as the Docker CLI, splitting out into a `domainname` option. + - TTY behaviour is far more robust, and resizes are supported correctly. + - Load YAML files safely. + +Thanks to @d11wtq, @ryanbrainard, @rail44, @j0hnsmith, @binarin, @Elemecca, @mozz100 and @marksteve for their help with this release! + + +0.4.2 (2014-06-18) +------------------ + + - Fix various encoding errors when using `fig run`, `fig up` and `fig build`. + +0.4.1 (2014-05-08) +------------------ + + - Add support for Docker 0.11.0. (Thanks @marksteve!) + - Make project name configurable. (Thanks @jefmathiot!) + - Return correct exit code from `fig run`. + +0.4.0 (2014-04-29) +------------------ + + - Support Docker 0.9 and 0.10 + - Display progress bars correctly when pulling images (no more ski slopes) + - `fig up` now stops all services when any container exits + - Added support for the `privileged` config option in fig.yml (thanks @kvz!) + - Shortened and aligned log prefixes in `fig up` output + - Only containers started with `fig run` link back to their own service + - Handle UTF-8 correctly when streaming `fig build/run/up` output (thanks @mauvm and @shanejonas!) + - Error message improvements + +0.3.2 (2014-03-05) +------------------ + + - Added an `--rm` option to `fig run`. (Thanks @marksteve!) + - Added an `expose` option to `fig.yml`. + +0.3.1 (2014-03-04) +------------------ + + - Added contribution instructions. (Thanks @kvz!) + - Fixed `fig rm` throwing an error. + - Fixed a bug in `fig ps` on Docker 0.8.1 when there is a container with no command. + +0.3.0 (2014-03-03) +------------------ + + - We now ship binaries for OS X and Linux. No more having to install with Pip! + - Add `-f` flag to specify alternate `fig.yml` files + - Add support for custom link names + - Fix a bug where recreating would sometimes hang + - Update docker-py to support Docker 0.8.0. + - Various documentation improvements + - Various error message improvements + +Thanks @marksteve, @Gazler and @teozkr! + +0.2.2 (2014-02-17) +------------------ + + - Resolve dependencies using Cormen/Tarjan topological sort + - Fix `fig up` not printing log output + - Stop containers in reverse order to starting + - Fix scale command not binding ports + +Thanks to @barnybug and @dustinlacewell for their work on this release. + +0.2.1 (2014-02-04) +------------------ + + - General improvements to error reporting (#77, #79) + +0.2.0 (2014-01-31) +------------------ + + - Link services to themselves so run commands can access the running service. (#67) + - Much better documentation. + - Make service dependency resolution more reliable. (#48) + - Load Fig configurations with a `.yaml` extension. (#58) + +Big thanks to @cameronmaske, @mrchrisadams and @damianmoore for their help with this release. + +0.1.4 (2014-01-27) +------------------ + + - Add a link alias without the project name. This makes the environment variables a little shorter: `REDIS_1_PORT_6379_TCP_ADDR`. (#54) + +0.1.3 (2014-01-23) +------------------ + + - Fix ports sometimes being configured incorrectly. (#46) + - Fix log output sometimes not displaying. (#47) + +0.1.2 (2014-01-22) +------------------ + + - Add `-T` option to `fig run` to disable pseudo-TTY. (#34) + - Fix `fig up` requiring the ubuntu image to be pulled to recreate containers. (#33) Thanks @cameronmaske! + - Improve reliability, fix arrow keys and fix a race condition in `fig run`. (#34, #39, #40) + +0.1.1 (2014-01-17) +------------------ + + - Fix bug where ports were not exposed correctly (#29). Thanks @dustinlacewell! + +0.1.0 (2014-01-16) +------------------ + + - Containers are recreated on each `fig up`, ensuring config is up-to-date with `fig.yml` (#2) + - Add `fig scale` command (#9) + - Use `DOCKER_HOST` environment variable to find Docker daemon, for consistency with the official Docker client (was previously `DOCKER_URL`) (#19) + - Truncate long commands in `fig ps` (#18) + - Fill out CLI help banners for commands (#15, #16) + - Show a friendlier error when `fig.yml` is missing (#4) + - Fix bug with `fig build` logging (#3) + - Fix bug where builds would time out if a step took a long time without generating output (#6) + - Fix bug where streaming container output over the Unix socket raised an error (#7) + +Big thanks to @tomstuart, @EnTeQuAk, @schickling, @aronasorman and @GeoffreyPlitt. + +0.0.2 (2014-01-02) +------------------ + + - Improve documentation + - Try to connect to Docker on `tcp://localdocker:4243` and a UNIX socket in addition to `localhost`. + - Improve `fig up` behaviour + - Add confirmation prompt to `fig rm` + - Add `fig build` command + +0.0.1 (2013-12-20) +------------------ + +Initial release. + + diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index 88e725da61..0000000000 --- a/CHANGES.md +++ /dev/null @@ -1,414 +0,0 @@ -Change log -========== - -1.4.0 (2015-08-04) ------------------- - -- By default, `docker-compose up` now only recreates containers for services whose configuration has changed since they were created. This should result in a dramatic speed-up for many applications. - - The experimental `--x-smart-recreate` flag which introduced this feature in Compose 1.3.0 has been removed, and a `--force-recreate` flag has been added for when you want to recreate everything. - -- Several of Compose's commands - `scale`, `stop`, `kill` and `rm` - now perform actions on multiple containers in parallel, rather than in sequence, which will run much faster on larger applications. - -- You can now specify a custom name for a service's container with `container_name`. Because Docker container names must be unique, this means you can't scale the service beyond one container. - -- You no longer have to specify a `file` option when using `extends` - it will default to the current file. - -- Service names can now contain dots, dashes and underscores. - -- Compose can now read YAML configuration from standard input, rather than from a file, by specifying `-` as the filename. This makes it easier to generate configuration dynamically: - - $ echo 'redis: {"image": "redis"}' | docker-compose --file - up - -- There's a new `docker-compose version` command which prints extended information about Compose's bundled dependencies. - -- `docker-compose.yml` now supports `log_opt` as well as `log_driver`, allowing you to pass extra configuration to a service's logging driver. - -- `docker-compose.yml` now supports `memswap_limit`, similar to `docker run --memory-swap`. - -- When mounting volumes with the `volumes` option, you can now pass in any mode supported by the daemon, not just `:ro` or `:rw`. For example, SELinux users can pass `:z` or `:Z`. - -- You can now specify a custom volume driver with the `volume_driver` option in `docker-compose.yml`, much like `docker run --volume-driver`. - -- A bug has been fixed where Compose would fail to pull images from private registries serving plain (unsecured) HTTP. The `--allow-insecure-ssl` flag, which was previously used to work around this issue, has been deprecated and now has no effect. - -- A bug has been fixed where `docker-compose build` would fail if the build depended on a private Hub image or an image from a private registry. - -- A bug has been fixed where Compose would crash if there were containers which the Docker daemon had not finished removing. - -- Two bugs have been fixed where Compose would sometimes fail with a "Duplicate bind mount" error, or fail to attach volumes to a container, if there was a volume path specified in `docker-compose.yml` with a trailing slash. - -Thanks @mnowster, @dnephin, @ekristen, @funkyfuture, @jeffk and @lukemarsden! - -1.3.3 (2015-07-15) ------------------- - -Two regressions have been fixed: - -- When stopping containers gracefully, Compose was setting the timeout to 0, effectively forcing a SIGKILL every time. -- Compose would sometimes crash depending on the formatting of container data returned from the Docker API. - -1.3.2 (2015-07-14) ------------------- - -The following bugs have been fixed: - -- When there were one-off containers created by running `docker-compose run` on an older version of Compose, `docker-compose run` would fail with a name collision. Compose now shows an error if you have leftover containers of this type lying around, and tells you how to remove them. -- Compose was not reading Docker authentication config files created in the new location, `~/docker/config.json`, and authentication against private registries would therefore fail. -- When a container had a pseudo-TTY attached, its output in `docker-compose up` would be truncated. -- `docker-compose up --x-smart-recreate` would sometimes fail when an image tag was updated. -- `docker-compose up` would sometimes create two containers with the same numeric suffix. -- `docker-compose rm` and `docker-compose ps` would sometimes list services that aren't part of the current project (though no containers were erroneously removed). -- Some `docker-compose` commands would not show an error if invalid service names were passed in. - -Thanks @dano, @josephpage, @kevinsimper, @lieryan, @phemmer, @soulrebel and @sschepens! - -1.3.1 (2015-06-21) ------------------- - -The following bugs have been fixed: - -- `docker-compose build` would always attempt to pull the base image before building. -- `docker-compose help migrate-to-labels` failed with an error. -- If no network mode was specified, Compose would set it to "bridge", rather than allowing the Docker daemon to use its configured default network mode. - -1.3.0 (2015-06-18) ------------------- - -Firstly, two important notes: - -- **This release contains breaking changes, and you will need to either remove or migrate your existing containers before running your app** - see the [upgrading section of the install docs](https://github.com/docker/compose/blob/1.3.0rc1/docs/install.md#upgrading) for details. - -- Compose now requires Docker 1.6.0 or later. - -We've done a lot of work in this release to remove hacks and make Compose more stable: - -- Compose now uses container labels, rather than names, to keep track of containers. This makes Compose both faster and easier to integrate with your own tools. - -- Compose no longer uses "intermediate containers" when recreating containers for a service. This makes `docker-compose up` less complex and more resilient to failure. - -There are some new features: - -- `docker-compose up` has an **experimental** new behaviour: it will only recreate containers for services whose configuration has changed in `docker-compose.yml`. This will eventually become the default, but for now you can take it for a spin: - - $ docker-compose up --x-smart-recreate - -- When invoked in a subdirectory of a project, `docker-compose` will now climb up through parent directories until it finds a `docker-compose.yml`. - -Several new configuration keys have been added to `docker-compose.yml`: - -- `dockerfile`, like `docker build --file`, lets you specify an alternate Dockerfile to use with `build`. -- `labels`, like `docker run --labels`, lets you add custom metadata to containers. -- `extra_hosts`, like `docker run --add-host`, lets you add entries to a container's `/etc/hosts` file. -- `pid: host`, like `docker run --pid=host`, lets you reuse the same PID namespace as the host machine. -- `cpuset`, like `docker run --cpuset-cpus`, lets you specify which CPUs to allow execution in. -- `read_only`, like `docker run --read-only`, lets you mount a container's filesystem as read-only. -- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/reference/run/#security-configuration). -- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/reference/run/#logging-drivers-log-driver). - -Many bugs have been fixed, including the following: - -- The output of `docker-compose run` was sometimes truncated, especially when running under Jenkins. -- A service's volumes would sometimes not update after volume configuration was changed in `docker-compose.yml`. -- Authenticating against third-party registries would sometimes fail. -- `docker-compose run --rm` would fail to remove the container if the service had a `restart` policy in place. -- `docker-compose scale` would refuse to scale a service beyond 1 container if it exposed a specific port number on the host. -- Compose would refuse to create multiple volume entries with the same host path. - -Thanks @ahromis, @albers, @aleksandr-vin, @antoineco, @ccverak, @chernjie, @dnephin, @edmorley, @fordhurley, @josephpage, @KyleJamesWalker, @lsowen, @mchasal, @noironetworks, @sdake, @sdurrheimer, @sherter, @stephenlawrence, @thaJeztah, @thieman, @turtlemonvh, @twhiteman, @vdemeester, @xuxinkun and @zwily! - -1.2.0 (2015-04-16) ------------------- - -- `docker-compose.yml` now supports an `extends` option, which enables a service to inherit configuration from another service in another configuration file. This is really good for sharing common configuration between apps, or for configuring the same app for different environments. Here's the [documentation](https://github.com/docker/compose/blob/master/docs/yml.md#extends). - -- When using Compose with a Swarm cluster, containers that depend on one another will be co-scheduled on the same node. This means that most Compose apps will now work out of the box, as long as they don't use `build`. - -- Repeated invocations of `docker-compose up` when using Compose with a Swarm cluster now work reliably. - -- Directories passed to `build`, filenames passed to `env_file` and volume host paths passed to `volumes` are now treated as relative to the *directory of the configuration file*, not the directory that `docker-compose` is being run in. In the majority of cases, those are the same, but if you use the `-f|--file` argument to specify a configuration file in another directory, **this is a breaking change**. - -- A service can now share another service's network namespace with `net: container:`. - -- `volumes_from` and `net: container:` entries are taken into account when resolving dependencies, so `docker-compose up ` will correctly start all dependencies of ``. - -- `docker-compose run` now accepts a `--user` argument to specify a user to run the command as, just like `docker run`. - -- The `up`, `stop` and `restart` commands now accept a `--timeout` (or `-t`) argument to specify how long to wait when attempting to gracefully stop containers, just like `docker stop`. - -- `docker-compose rm` now accepts `-f` as a shorthand for `--force`, just like `docker rm`. - -Thanks, @abesto, @albers, @alunduil, @dnephin, @funkyfuture, @gilclark, @IanVS, @KingsleyKelly, @knutwalker, @thaJeztah and @vmalloc! - -1.1.0 (2015-02-25) ------------------- - -Fig has been renamed to Docker Compose, or just Compose for short. This has several implications for you: - -- The command you type is now `docker-compose`, not `fig`. -- You should rename your fig.yml to docker-compose.yml. -- If you’re installing via PyPi, the package is now `docker-compose`, so install it with `pip install docker-compose`. - -Besides that, there’s a lot of new stuff in this release: - -- We’ve made a few small changes to ensure that Compose will work with Swarm, Docker’s new clustering tool (https://github.com/docker/swarm). Eventually you'll be able to point Compose at a Swarm cluster instead of a standalone Docker host and it’ll run your containers on the cluster with no extra work from you. As Swarm is still developing, integration is rough and lots of Compose features don't work yet. - -- `docker-compose run` now has a `--service-ports` flag for exposing ports on the given service. This is useful for e.g. running your webapp with an interactive debugger. - -- You can now link to containers outside your app with the `external_links` option in docker-compose.yml. - -- You can now prevent `docker-compose up` from automatically building images with the `--no-build` option. This will make fewer API calls and run faster. - -- If you don’t specify a tag when using the `image` key, Compose will default to the `latest` tag, rather than pulling all tags. - -- `docker-compose kill` now supports the `-s` flag, allowing you to specify the exact signal you want to send to a service’s containers. - -- docker-compose.yml now has an `env_file` key, analogous to `docker run --env-file`, letting you specify multiple environment variables in a separate file. This is great if you have a lot of them, or if you want to keep sensitive information out of version control. - -- docker-compose.yml now supports the `dns_search`, `cap_add`, `cap_drop`, `cpu_shares` and `restart` options, analogous to `docker run`’s `--dns-search`, `--cap-add`, `--cap-drop`, `--cpu-shares` and `--restart` options. - -- Compose now ships with Bash tab completion - see the installation and usage docs at https://github.com/docker/compose/blob/1.1.0/docs/completion.md - -- A number of bugs have been fixed - see the milestone for details: https://github.com/docker/compose/issues?q=milestone%3A1.1.0+ - -Thanks @dnephin, @squebe, @jbalonso, @raulcd, @benlangfield, @albers, @ggtools, @bersace, @dtenenba, @petercv, @drewkett, @TFenby, @paulRbr, @Aigeruth and @salehe! - -1.0.1 (2014-11-04) ------------------- - - - Added an `--allow-insecure-ssl` option to allow `fig up`, `fig run` and `fig pull` to pull from insecure registries. - - Fixed `fig run` not showing output in Jenkins. - - Fixed a bug where Fig couldn't build Dockerfiles with ADD statements pointing at URLs. - -1.0.0 (2014-10-16) ------------------- - -The highlights: - - - [Fig has joined Docker.](https://www.orchardup.com/blog/orchard-is-joining-docker) Fig will continue to be maintained, but we'll also be incorporating the best bits of Fig into Docker itself. - - This means the GitHub repository has moved to [https://github.com/docker/fig](https://github.com/docker/fig) and our IRC channel is now #docker-fig on Freenode. - - - Fig can be used with the [official Docker OS X installer](https://docs.docker.com/installation/mac/). Boot2Docker will mount the home directory from your host machine so volumes work as expected. - - - Fig supports Docker 1.3. - - - It is now possible to connect to the Docker daemon using TLS by using the `DOCKER_CERT_PATH` and `DOCKER_TLS_VERIFY` environment variables. - - - There is a new `fig port` command which outputs the host port binding of a service, in a similar way to `docker port`. - - - There is a new `fig pull` command which pulls the latest images for a service. - - - There is a new `fig restart` command which restarts a service's containers. - - - Fig creates multiple containers in service by appending a number to the service name (e.g. `db_1`, `db_2`, etc). As a convenience, Fig will now give the first container an alias of the service name (e.g. `db`). - - This link alias is also a valid hostname and added to `/etc/hosts` so you can connect to linked services using their hostname. For example, instead of resolving the environment variables `DB_PORT_5432_TCP_ADDR` and `DB_PORT_5432_TCP_PORT`, you could just use the hostname `db` and port `5432` directly. - - - Volume definitions now support `ro` mode, expanding `~` and expanding environment variables. - - - `.dockerignore` is supported when building. - - - The project name can be set with the `FIG_PROJECT_NAME` environment variable. - - - The `--env` and `--entrypoint` options have been added to `fig run`. - - - The Fig binary for Linux is now linked against an older version of glibc so it works on CentOS 6 and Debian Wheezy. - -Other things: - - - `fig ps` now works on Jenkins and makes fewer API calls to the Docker daemon. - - `--verbose` displays more useful debugging output. - - When starting a service where `volumes_from` points to a service without any containers running, that service will now be started. - - Lots of docs improvements. Notably, environment variables are documented and official repositories are used throughout. - -Thanks @dnephin, @d11wtq, @marksteve, @rubbish, @jbalonso, @timfreund, @alunduil, @mieciu, @shuron, @moss, @suzaku and @chmouel! Whew. - -0.5.2 (2014-07-28) ------------------- - - - Added a `--no-cache` option to `fig build`, which bypasses the cache just like `docker build --no-cache`. - - Fixed the `dns:` fig.yml option, which was causing fig to error out. - - Fixed a bug where fig couldn't start under Python 2.6. - - Fixed a log-streaming bug that occasionally caused fig to exit. - -Thanks @dnephin and @marksteve! - - -0.5.1 (2014-07-11) ------------------- - - - If a service has a command defined, `fig run [service]` with no further arguments will run it. - - The project name now defaults to the directory containing fig.yml, not the current working directory (if they're different) - - `volumes_from` now works properly with containers as well as services - - Fixed a race condition when recreating containers in `fig up` - -Thanks @ryanbrainard and @d11wtq! - - -0.5.0 (2014-07-11) ------------------- - - - Fig now starts links when you run `fig run` or `fig up`. - - For example, if you have a `web` service which depends on a `db` service, `fig run web ...` will start the `db` service. - - - Environment variables can now be resolved from the environment that Fig is running in. Just specify it as a blank variable in your `fig.yml` and, if set, it'll be resolved: - ``` - environment: - RACK_ENV: development - SESSION_SECRET: - ``` - - - `volumes_from` is now supported in `fig.yml`. All of the volumes from the specified services and containers will be mounted: - - ``` - volumes_from: - - service_name - - container_name - ``` - - - A host address can now be specified in `ports`: - - ``` - ports: - - "0.0.0.0:8000:8000" - - "127.0.0.1:8001:8001" - ``` - - - The `net` and `workdir` options are now supported in `fig.yml`. - - The `hostname` option now works in the same way as the Docker CLI, splitting out into a `domainname` option. - - TTY behaviour is far more robust, and resizes are supported correctly. - - Load YAML files safely. - -Thanks to @d11wtq, @ryanbrainard, @rail44, @j0hnsmith, @binarin, @Elemecca, @mozz100 and @marksteve for their help with this release! - - -0.4.2 (2014-06-18) ------------------- - - - Fix various encoding errors when using `fig run`, `fig up` and `fig build`. - -0.4.1 (2014-05-08) ------------------- - - - Add support for Docker 0.11.0. (Thanks @marksteve!) - - Make project name configurable. (Thanks @jefmathiot!) - - Return correct exit code from `fig run`. - -0.4.0 (2014-04-29) ------------------- - - - Support Docker 0.9 and 0.10 - - Display progress bars correctly when pulling images (no more ski slopes) - - `fig up` now stops all services when any container exits - - Added support for the `privileged` config option in fig.yml (thanks @kvz!) - - Shortened and aligned log prefixes in `fig up` output - - Only containers started with `fig run` link back to their own service - - Handle UTF-8 correctly when streaming `fig build/run/up` output (thanks @mauvm and @shanejonas!) - - Error message improvements - -0.3.2 (2014-03-05) ------------------- - - - Added an `--rm` option to `fig run`. (Thanks @marksteve!) - - Added an `expose` option to `fig.yml`. - -0.3.1 (2014-03-04) ------------------- - - - Added contribution instructions. (Thanks @kvz!) - - Fixed `fig rm` throwing an error. - - Fixed a bug in `fig ps` on Docker 0.8.1 when there is a container with no command. - -0.3.0 (2014-03-03) ------------------- - - - We now ship binaries for OS X and Linux. No more having to install with Pip! - - Add `-f` flag to specify alternate `fig.yml` files - - Add support for custom link names - - Fix a bug where recreating would sometimes hang - - Update docker-py to support Docker 0.8.0. - - Various documentation improvements - - Various error message improvements - -Thanks @marksteve, @Gazler and @teozkr! - -0.2.2 (2014-02-17) ------------------- - - - Resolve dependencies using Cormen/Tarjan topological sort - - Fix `fig up` not printing log output - - Stop containers in reverse order to starting - - Fix scale command not binding ports - -Thanks to @barnybug and @dustinlacewell for their work on this release. - -0.2.1 (2014-02-04) ------------------- - - - General improvements to error reporting (#77, #79) - -0.2.0 (2014-01-31) ------------------- - - - Link services to themselves so run commands can access the running service. (#67) - - Much better documentation. - - Make service dependency resolution more reliable. (#48) - - Load Fig configurations with a `.yaml` extension. (#58) - -Big thanks to @cameronmaske, @mrchrisadams and @damianmoore for their help with this release. - -0.1.4 (2014-01-27) ------------------- - - - Add a link alias without the project name. This makes the environment variables a little shorter: `REDIS_1_PORT_6379_TCP_ADDR`. (#54) - -0.1.3 (2014-01-23) ------------------- - - - Fix ports sometimes being configured incorrectly. (#46) - - Fix log output sometimes not displaying. (#47) - -0.1.2 (2014-01-22) ------------------- - - - Add `-T` option to `fig run` to disable pseudo-TTY. (#34) - - Fix `fig up` requiring the ubuntu image to be pulled to recreate containers. (#33) Thanks @cameronmaske! - - Improve reliability, fix arrow keys and fix a race condition in `fig run`. (#34, #39, #40) - -0.1.1 (2014-01-17) ------------------- - - - Fix bug where ports were not exposed correctly (#29). Thanks @dustinlacewell! - -0.1.0 (2014-01-16) ------------------- - - - Containers are recreated on each `fig up`, ensuring config is up-to-date with `fig.yml` (#2) - - Add `fig scale` command (#9) - - Use `DOCKER_HOST` environment variable to find Docker daemon, for consistency with the official Docker client (was previously `DOCKER_URL`) (#19) - - Truncate long commands in `fig ps` (#18) - - Fill out CLI help banners for commands (#15, #16) - - Show a friendlier error when `fig.yml` is missing (#4) - - Fix bug with `fig build` logging (#3) - - Fix bug where builds would time out if a step took a long time without generating output (#6) - - Fix bug where streaming container output over the Unix socket raised an error (#7) - -Big thanks to @tomstuart, @EnTeQuAk, @schickling, @aronasorman and @GeoffreyPlitt. - -0.0.2 (2014-01-02) ------------------- - - - Improve documentation - - Try to connect to Docker on `tcp://localdocker:4243` and a UNIX socket in addition to `localhost`. - - Improve `fig up` behaviour - - Add confirmation prompt to `fig rm` - - Add `fig build` command - -0.0.1 (2013-12-20) ------------------- - -Initial release. - - diff --git a/CHANGES.md b/CHANGES.md new file mode 120000 index 0000000000..83b694704b --- /dev/null +++ b/CHANGES.md @@ -0,0 +1 @@ +CHANGELOG.md \ No newline at end of file From 65afce526a83c6654428d0d06e45a25991a755fd Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 14 Aug 2015 12:42:33 +0100 Subject: [PATCH 062/337] Test against Docker 1.8.1 Signed-off-by: Aanand Prasad --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c0482323b..ed23e75acb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,13 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.0-rc3 +ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.1 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.7.1 -o /usr/local/bin/docker-1.7.1; \ chmod +x /usr/local/bin/docker-1.7.1; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.8.0-rc3 -o /usr/local/bin/docker-1.8.0-rc3; \ - chmod +x /usr/local/bin/docker-1.8.0-rc3 + curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.1 -o /usr/local/bin/docker-1.8.1; \ + chmod +x /usr/local/bin/docker-1.8.1 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker From f4a8fda283ee63c524b6929d11c71eac4be3751c Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 17 Aug 2015 16:31:57 +0100 Subject: [PATCH 063/337] Handle all exceptions If we get back an error that wasn't an APIError, it was causing the thread to hang. This catch all, while I appreciate feels risky to have a catch all, is better than not catching and silently failing, with a never ending thread. If something worse than an APIError has gone wrong, we want to stop the incredible journey of what we're doing. Signed-off-by: Mazz Mosley --- compose/utils.py | 7 +++++++ tests/integration/service_test.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/compose/utils.py b/compose/utils.py index 4c7f94c576..61d6d80243 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -32,6 +32,10 @@ def parallel_execute(objects, obj_callable, msg_index, msg): except APIError as e: errors[msg_index] = e.explanation result = "error" + except Exception as e: + errors[msg_index] = e + result = 'unexpected_exception' + q.put((msg_index, result)) for an_object in objects: @@ -48,6 +52,9 @@ def parallel_execute(objects, obj_callable, msg_index, msg): while done < total_to_execute: try: msg_index, result = q.get(timeout=1) + + if result == 'unexpected_exception': + raise errors[msg_index] if result == 'error': write_out_msg(stream, lines, msg_index, msg, status='error') else: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 9bdc12f993..050a3bf622 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -654,6 +654,25 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(service.containers()[0].is_running) self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) + @patch('sys.stdout', new_callable=StringIO) + def test_scale_with_api_returns_unexpected_exception(self, mock_stdout): + """ + Test that when scaling if the API returns an error, that is not of type + APIError, that error is re-raised. + """ + service = self.create_service('web') + next_number = service._next_container_number() + service.create_container(number=next_number, quiet=True) + + with patch( + 'compose.container.Container.create', + side_effect=ValueError("BOOM")): + with self.assertRaises(ValueError): + service.scale(3) + + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + @patch('compose.service.log') def test_scale_with_desired_number_already_achieved(self, mock_log): """ From 18a474211d29a59b9251ce4aa9947bd7d8241114 Mon Sep 17 00:00:00 2001 From: Maxime Horcholle Date: Tue, 18 Aug 2015 09:07:15 +0200 Subject: [PATCH 064/337] remove extra ``` Signed-off-by: mhor --- docs/yml.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/yml.md b/docs/yml.md index bec857f8e2..3e9a35ca43 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -394,7 +394,6 @@ Each of these is a single value, analogous to its read_only: true volume_driver: mydriver -``` ## Variable substitution From 56f03bc20acaffc9b5d4c2a5898ef47126f4df19 Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Tue, 18 Aug 2015 21:46:05 +0100 Subject: [PATCH 065/337] Allow to specify image by digest. Fixes #1670 Signed-off-by: Karol Duleba --- compose/service.py | 35 +++++++++++++++----- docs/yml.md | 3 +- tests/fixtures/simple-composefile/digest.yml | 6 ++++ tests/integration/cli_test.py | 6 ++++ tests/unit/service_test.py | 26 +++++++++++---- 5 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 tests/fixtures/simple-composefile/digest.yml diff --git a/compose/service.py b/compose/service.py index 5a79414bcd..e49acf0c6a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -757,9 +757,9 @@ class Service(object): if 'image' not in self.options: return - repo, tag = parse_repository_tag(self.options['image']) + repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' - log.info('Pulling %s (%s:%s)...' % (self.name, repo, tag)) + log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) output = self.client.pull( repo, tag=tag, @@ -780,14 +780,31 @@ def build_container_name(project, service, number, one_off=False): # Images +def parse_repository_tag(repo_path): + """Splits image identification into base image path, tag/digest + and it's separator. -def parse_repository_tag(s): - if ":" not in s: - return s, "" - repo, tag = s.rsplit(":", 1) - if "/" in tag: - return s, "" - return repo, tag + Example: + + >>> parse_repository_tag('user/repo@sha256:digest') + ('user/repo', 'sha256:digest', '@') + >>> parse_repository_tag('user/repo:v1') + ('user/repo', 'v1', ':') + """ + tag_separator = ":" + digest_separator = "@" + + if digest_separator in repo_path: + repo, tag = repo_path.rsplit(digest_separator, 1) + return repo, tag, digest_separator + + repo, tag = repo_path, "" + if tag_separator in repo_path: + repo, tag = repo_path.rsplit(tag_separator, 1) + if "/" in tag: + repo, tag = repo_path, "" + + return repo, tag, tag_separator # Volumes diff --git a/docs/yml.md b/docs/yml.md index 3e9a35ca43..bad9c9bc19 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -25,12 +25,13 @@ Values for configuration options can contain environment variables, e.g. ### image -Tag or partial image ID. Can be local or remote - Compose will attempt to +Tag, partial image ID or digest. Can be local or remote - Compose will attempt to pull if it doesn't exist locally. image: ubuntu image: orchardup/postgresql image: a4bc65fd + image: busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d ### build diff --git a/tests/fixtures/simple-composefile/digest.yml b/tests/fixtures/simple-composefile/digest.yml new file mode 100644 index 0000000000..08f1d993e9 --- /dev/null +++ b/tests/fixtures/simple-composefile/digest.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: top +digest: + image: busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d + command: top diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index ef789e19c7..a02e072fd7 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -88,6 +88,12 @@ class CLITestCase(DockerClientTestCase): mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') mock_logging.info.assert_any_call('Pulling another (busybox:latest)...') + @patch('compose.service.log') + def test_pull_with_digest(self, mock_logging): + self.command.dispatch(['-f', 'digest.yml', 'pull'], None) + mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') + mock_logging.info.assert_any_call('Pulling digest (busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') + @patch('sys.stdout', new_callable=StringIO) def test_build_no_cache(self, mock_stdout): self.command.base_dir = 'tests/fixtures/simple-dockerfile' diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index bb68c9aa6d..8b39a63ef7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -192,6 +192,16 @@ class ServiceTest(unittest.TestCase): tag='latest', stream=True) + @mock.patch('compose.service.log', autospec=True) + def test_pull_image_digest(self, mock_log): + service = Service('foo', client=self.mock_client, image='someimage@sha256:1234') + service.pull() + self.mock_client.pull.assert_called_once_with( + 'someimage', + tag='sha256:1234', + stream=True) + mock_log.info.assert_called_once_with('Pulling foo (someimage@sha256:1234)...') + @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) @@ -217,12 +227,16 @@ class ServiceTest(unittest.TestCase): mock_container.stop.assert_called_once_with(timeout=1) def test_parse_repository_tag(self): - self.assertEqual(parse_repository_tag("root"), ("root", "")) - self.assertEqual(parse_repository_tag("root:tag"), ("root", "tag")) - self.assertEqual(parse_repository_tag("user/repo"), ("user/repo", "")) - self.assertEqual(parse_repository_tag("user/repo:tag"), ("user/repo", "tag")) - self.assertEqual(parse_repository_tag("url:5000/repo"), ("url:5000/repo", "")) - self.assertEqual(parse_repository_tag("url:5000/repo:tag"), ("url:5000/repo", "tag")) + self.assertEqual(parse_repository_tag("root"), ("root", "", ":")) + self.assertEqual(parse_repository_tag("root:tag"), ("root", "tag", ":")) + self.assertEqual(parse_repository_tag("user/repo"), ("user/repo", "", ":")) + self.assertEqual(parse_repository_tag("user/repo:tag"), ("user/repo", "tag", ":")) + self.assertEqual(parse_repository_tag("url:5000/repo"), ("url:5000/repo", "", ":")) + self.assertEqual(parse_repository_tag("url:5000/repo:tag"), ("url:5000/repo", "tag", ":")) + + self.assertEqual(parse_repository_tag("root@sha256:digest"), ("root", "sha256:digest", "@")) + self.assertEqual(parse_repository_tag("user/repo@sha256:digest"), ("user/repo", "sha256:digest", "@")) + self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) @mock.patch('compose.service.Container', autospec=True) def test_create_container_latest_is_used_when_no_tag_specified(self, mock_container): From 61936f6b88e8e859b23dcf933246675185aaeabd Mon Sep 17 00:00:00 2001 From: Joel Hansson Date: Thu, 20 Aug 2015 16:38:43 +0200 Subject: [PATCH 066/337] log_opt: change address to syslog-address Signed-off-by: Joel Hansson --- compose/config/schema.json | 4 ++-- docs/yml.md | 2 +- tests/unit/service_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 073a0da65a..17e1445a65 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -62,9 +62,9 @@ "type": "object", "properties": { - "address": {"type": "string"} + "syslog-address": {"type": "string"} }, - "required": ["address"] + "required": ["syslog-address"] }, "mac_address": {"type": "string"}, diff --git a/docs/yml.md b/docs/yml.md index bad9c9bc19..9662208641 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -301,7 +301,7 @@ Logging options are key value pairs. An example of `syslog` options: log_driver: "syslog" log_opt: - address: "tcp://192.168.0.42:123" + syslog-address: "tcp://192.168.0.42:123" ### net diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8b39a63ef7..2965d6c891 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -113,7 +113,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['host_config']['Memory'], 1000000000) def test_log_opt(self): - log_opt = {'address': 'tcp://192.168.0.42:123'} + log_opt = {'syslog-address': 'tcp://192.168.0.42:123'} service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'some': 'overrides'}, 1) From c69987661728528655e06d12a8ab76528590192c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 20 Aug 2015 16:09:28 +0100 Subject: [PATCH 067/337] Set log level to DEBUG when `--verbose` is passed Signed-off-by: Aanand Prasad --- compose/cli/main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6c2a8edb61..cb38f54c24 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -25,6 +25,7 @@ from .log_printer import LogPrinter from .utils import yesno, get_version_info log = logging.getLogger(__name__) +console_handler = logging.StreamHandler(sys.stderr) INSECURE_SSL_WARNING = """ Warning: --allow-insecure-ssl is deprecated and has no effect. @@ -63,9 +64,6 @@ def main(): def setup_logging(): - console_handler = logging.StreamHandler(sys.stderr) - console_handler.setFormatter(logging.Formatter()) - console_handler.setLevel(logging.INFO) root_logger = logging.getLogger() root_logger.addHandler(console_handler) root_logger.setLevel(logging.DEBUG) @@ -118,6 +116,16 @@ class TopLevelCommand(Command): options['version'] = get_version_info('compose') return options + def perform_command(self, options, *args, **kwargs): + if options.get('--verbose'): + console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) + console_handler.setLevel(logging.DEBUG) + else: + console_handler.setFormatter(logging.Formatter()) + console_handler.setLevel(logging.INFO) + + return super(TopLevelCommand, self).perform_command(options, *args, **kwargs) + def build(self, project, options): """ Build or rebuild services. From 8caaee9eac0032e74b44c8f65bea28fff73630ff Mon Sep 17 00:00:00 2001 From: Joel Hansson Date: Fri, 21 Aug 2015 08:36:03 +0200 Subject: [PATCH 068/337] schema.json: remove specific log_opt properties Signed-off-by: Joel Hansson --- compose/config/schema.json | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 17e1445a65..8e9b79fb64 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -56,16 +56,9 @@ "image": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "log_driver": {"type": "string"}, - - "log_opt": { - "type": "object", - - "properties": { - "syslog-address": {"type": "string"} - }, - "required": ["syslog-address"] - }, + "log_opt": {"type": "object"}, "mac_address": {"type": "string"}, "mem_limit": { From 227584b8640be269f60975d7c7f361e856c9e9f6 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 25 Jul 2015 22:20:58 +0200 Subject: [PATCH 069/337] Adds pause and unpause-commands Signed-off-by: Frank Sachsenheim --- compose/cli/main.py | 16 +++++++++++++ compose/container.py | 12 ++++++++++ compose/project.py | 8 +++++++ compose/service.py | 16 +++++++++++-- contrib/completion/bash/docker-compose | 31 ++++++++++++++++++++++++++ docs/reference/docker-compose.md | 2 ++ docs/reference/pause.md | 18 +++++++++++++++ docs/reference/unpause.md | 18 +++++++++++++++ tests/integration/cli_test.py | 11 +++++++++ tests/integration/project_test.py | 19 ++++++++++++++-- 10 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 docs/reference/pause.md create mode 100644 docs/reference/unpause.md diff --git a/compose/cli/main.py b/compose/cli/main.py index 6c2a8edb61..df0dfe9f34 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -172,6 +172,14 @@ class TopLevelCommand(Command): print("Attaching to", list_containers(containers)) LogPrinter(containers, attach_params={'logs': True}, monochrome=monochrome).run() + def pause(self, project, options): + """ + Pause services. + + Usage: pause [SERVICE...] + """ + project.pause(service_names=options['SERVICE']) + def port(self, project, options): """ Print the public port for a port binding. @@ -444,6 +452,14 @@ class TopLevelCommand(Command): timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) project.restart(service_names=options['SERVICE'], timeout=timeout) + def unpause(self, project, options): + """ + Unpause services. + + Usage: unpause [SERVICE...] + """ + project.unpause(service_names=options['SERVICE']) + def up(self, project, options): """ Builds, (re)creates, starts, and attaches to containers for a service. diff --git a/compose/container.py b/compose/container.py index 40aea98a45..37ed1fe593 100644 --- a/compose/container.py +++ b/compose/container.py @@ -100,6 +100,8 @@ class Container(object): @property def human_readable_state(self): + if self.is_paused: + return 'Paused' if self.is_running: return 'Ghost' if self.get('State.Ghost') else 'Up' else: @@ -119,6 +121,10 @@ class Container(object): def is_running(self): return self.get('State.Running') + @property + def is_paused(self): + return self.get('State.Paused') + def get(self, key): """Return a value from the container or None if the value is not set. @@ -142,6 +148,12 @@ class Container(object): def stop(self, **options): return self.client.stop(self.id, **options) + def pause(self, **options): + return self.client.pause(self.id, **options) + + def unpause(self, **options): + return self.client.unpause(self.id, **options) + def kill(self, **options): return self.client.kill(self.id, **options) diff --git a/compose/project.py b/compose/project.py index 6d86a4a872..276afb543f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -205,6 +205,14 @@ class Project(object): msg="Stopping" ) + def pause(self, service_names=None, **options): + for service in reversed(self.get_services(service_names)): + service.pause(**options) + + def unpause(self, service_names=None, **options): + for service in self.get_services(service_names): + service.unpause(**options) + def kill(self, service_names=None, **options): parallel_execute( objects=self.containers(service_names), diff --git a/compose/service.py b/compose/service.py index e49acf0c6a..28d289ffa7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -96,12 +96,14 @@ class Service(object): self.net = net or None self.options = options - def containers(self, stopped=False, one_off=False): + def containers(self, stopped=False, one_off=False, filters={}): + filters.update({'label': self.labels(one_off=one_off)}) + containers = filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters={'label': self.labels(one_off=one_off)})]) + filters=filters)]) if not containers: check_for_legacy_containers( @@ -132,6 +134,16 @@ class Service(object): log.info("Stopping %s..." % c.name) c.stop(**options) + def pause(self, **options): + for c in self.containers(filters={'status': 'running'}): + log.info("Pausing %s..." % c.name) + c.pause(**options) + + def unpause(self, **options): + for c in self.containers(filters={'status': 'paused'}): + log.info("Unpausing %s..." % c.name) + c.unpause() + def kill(self, **options): for c in self.containers(): log.info("Killing %s..." % c.name) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 66eb6c8bf1..5692f0e4b8 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -68,6 +68,11 @@ __docker_compose_services_with() { COMPREPLY=( $(compgen -W "${names[*]}" -- "$cur") ) } +# The services for which at least one paused container exists +__docker_compose_services_paused() { + __docker_compose_services_with '.State.Paused' +} + # The services for which at least one running container exists __docker_compose_services_running() { __docker_compose_services_with '.State.Running' @@ -158,6 +163,18 @@ _docker_compose_migrate_to_labels() { } +_docker_compose_pause() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_compose_services_running + ;; + esac +} + + _docker_compose_port() { case "$prev" in --protocol) @@ -306,6 +323,18 @@ _docker_compose_stop() { } +_docker_compose_unpause() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_compose_services_paused + ;; + esac +} + + _docker_compose_up() { case "$prev" in -t | --timeout) @@ -343,6 +372,7 @@ _docker_compose() { kill logs migrate-to-labels + pause port ps pull @@ -352,6 +382,7 @@ _docker_compose() { scale start stop + unpause up version ) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index e252da0a70..46afba13c1 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -28,6 +28,7 @@ Commands: help Get help on a command kill Kill containers logs View output from containers + pause Pause services port Print the public port for a port binding ps List containers pull Pulls service images @@ -37,6 +38,7 @@ Commands: scale Set number of containers for a service start Start services stop Stop services + unpause Unpause services up Create and start containers migrate-to-labels Recreate containers to add labels ``` diff --git a/docs/reference/pause.md b/docs/reference/pause.md new file mode 100644 index 0000000000..a0ffab0359 --- /dev/null +++ b/docs/reference/pause.md @@ -0,0 +1,18 @@ + + +# pause + +``` +Usage: pause [SERVICE...] +``` + +Pauses running containers of a service. They can be unpaused with `docker-compose unpause`. diff --git a/docs/reference/unpause.md b/docs/reference/unpause.md new file mode 100644 index 0000000000..6434b09ccc --- /dev/null +++ b/docs/reference/unpause.md @@ -0,0 +1,18 @@ + + +# pause + +``` +Usage: unpause [SERVICE...] +``` + +Unpauses paused containers of a service. diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index a02e072fd7..38f8ee4648 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -415,6 +415,17 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) + def test_pause_unpause(self): + self.command.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertFalse(service.containers()[0].is_paused) + + self.command.dispatch(['pause'], None) + self.assertTrue(service.containers()[0].is_paused) + + self.command.dispatch(['unpause'], None) + self.assertFalse(service.containers()[0].is_paused) + def test_logs_invalid_service_name(self): with self.assertRaises(NoSuchService): self.command.dispatch(['logs', 'madeupname'], None) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 9788c18615..ad2fe4fea1 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -140,7 +140,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') self.assertEqual(web._get_net(), 'container:' + net_container.id) - def test_start_stop_kill_remove(self): + def test_start_pause_unpause_stop_kill_remove(self): web = self.create_service('web') db = self.create_service('db') project = Project('composetest', [web, db], self.client) @@ -158,7 +158,22 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name])) project.start() - self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name, db_container.name])) + self.assertEqual(set(c.name for c in project.containers()), + set([web_container_1.name, web_container_2.name, db_container.name])) + + project.pause(service_names=['web']) + self.assertEqual(set([c.name for c in project.containers() if c.is_paused]), + set([web_container_1.name, web_container_2.name])) + + project.pause() + self.assertEqual(set([c.name for c in project.containers() if c.is_paused]), + set([web_container_1.name, web_container_2.name, db_container.name])) + + project.unpause(service_names=['db']) + self.assertEqual(len([c.name for c in project.containers() if c.is_paused]), 2) + + project.unpause() + self.assertEqual(len([c.name for c in project.containers() if c.is_paused]), 0) project.stop(service_names=['web'], timeout=1) self.assertEqual(set(c.name for c in project.containers()), set([db_container.name])) From dd738b380b43387724909e3e6caad863c8a9d6e0 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 25 Jul 2015 22:48:55 +0200 Subject: [PATCH 070/337] Makes Service.config_hash a property Signed-off-by: Frank Sachsenheim --- compose/service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 28d289ffa7..7df5618c55 100644 --- a/compose/service.py +++ b/compose/service.py @@ -343,7 +343,7 @@ class Service(object): config_hash = None try: - config_hash = self.config_hash() + config_hash = self.config_hash except NoSuchImageError as e: log.debug( 'Service %s has diverged: %s', @@ -468,6 +468,7 @@ class Service(object): else: numbers.add(c.number) + @property def config_hash(self): return json_hash(self.config_dict()) @@ -585,7 +586,7 @@ class Service(object): container_options['name'] = self.get_container_name(number, one_off) if add_config_hash: - config_hash = self.config_hash() + config_hash = self.config_hash if 'labels' not in container_options: container_options['labels'] = {} container_options['labels'][LABEL_CONFIG_HASH] = config_hash From a57ce1b1ba18750f6212055566a0f0007e44980e Mon Sep 17 00:00:00 2001 From: Berk Birand Date: Mon, 24 Aug 2015 15:10:00 -0400 Subject: [PATCH 071/337] Export COMPOSE_FILE The environment variable is not used by `docker-compose` without the `export` line.. Signed-off-by: Berk Birand --- docs/production.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/production.md b/docs/production.md index 6005113695..85f245810f 100644 --- a/docs/production.md +++ b/docs/production.md @@ -40,7 +40,7 @@ For this reason, you'll probably want to define a separate Compose file, say Once you've got an alternate configuration file, make Compose use it by setting the `COMPOSE_FILE` environment variable: - $ COMPOSE_FILE=production.yml + $ export COMPOSE_FILE=production.yml $ docker-compose up -d > **Note:** You can also use the file for a one-off command without setting From fae645466115045f3801cd3926afe752c08840ec Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 15:15:08 -0400 Subject: [PATCH 072/337] Add pre-commit hooks Signed-off-by: Daniel Nephin --- .pre-commit-config.yaml | 18 ++++++++++++++++++ Dockerfile | 3 +++ script/test-versions | 2 +- tox.ini | 12 +++++++++++- 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..832be6ab8d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: 'v0.4.2' + hooks: + - id: check-added-large-files + - id: check-docstring-first + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: flake8 + - id: name-tests-test + exclude: 'tests/integration/testcases.py' + - id: requirements-txt-fixer + - id: trailing-whitespace +- repo: git://github.com/asottile/reorder_python_imports + sha: 3d86483455ab5bd06cc1069fdd5ac57be5463f10 + hooks: + - id: reorder-python-imports diff --git a/Dockerfile b/Dockerfile index ed23e75acb..a4cc99fea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN set -ex; \ curl \ lxc \ iptables \ + libsqlite3-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -68,6 +69,8 @@ RUN pip install -r requirements.txt ADD requirements-dev.txt /code/ RUN pip install -r requirements-dev.txt +RUN pip install tox==2.1.1 + ADD . /code/ RUN python setup.py install diff --git a/script/test-versions b/script/test-versions index ae9620e384..d67a6f5e12 100755 --- a/script/test-versions +++ b/script/test-versions @@ -5,7 +5,7 @@ set -e >&2 echo "Running lint checks" -flake8 compose tests setup.py +tox -e pre-commit if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="default" diff --git a/tox.ini b/tox.ini index 33cdee167f..3a69c5784c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,10 @@ [tox] -envlist = py26,py27 +envlist = py27,pre-commit [testenv] usedevelop=True +passenv = + LD_LIBRARY_PATH deps = -rrequirements.txt -rrequirements-dev.txt @@ -10,6 +12,14 @@ commands = nosetests -v {posargs} flake8 compose tests setup.py +[testenv:pre-commit] +skip_install = True +deps = + pre-commit +commands = + pre-commit install + pre-commit run --all-files + [flake8] # ignore line-length for now ignore = E501,E203 From 59d4f304ee3bf4bb20ba0f5e0ad6c4a3ff1568f3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 15:25:25 -0400 Subject: [PATCH 073/337] Run pre-commit on all files Signed-off-by: Daniel Nephin --- CHANGELOG.md | 6 +-- README.md | 2 +- compose/cli/command.py | 21 ++++++---- compose/cli/docker_client.py | 5 ++- compose/cli/docopt_command.py | 8 ++-- compose/cli/errors.py | 1 + compose/cli/formatter.py | 4 +- compose/cli/log_printer.py | 6 +-- compose/cli/main.py | 18 +++++---- compose/cli/multiplexer.py | 1 + compose/cli/utils.py | 10 +++-- compose/cli/verbose_proxy.py | 3 +- compose/config/__init__.py | 19 +++++---- compose/config/config.py | 22 +++++----- compose/config/interpolation.py | 3 +- compose/config/validation.py | 8 ++-- compose/container.py | 8 ++-- compose/legacy.py | 3 +- compose/progress_stream.py | 2 +- compose/project.py | 13 ++++-- compose/service.py | 40 ++++++++++--------- compose/utils.py | 5 ++- docs/README.md | 12 +++--- docs/index.md | 2 +- docs/install.md | 16 ++++---- docs/pre-process.sh | 7 ++-- docs/production.md | 1 - docs/rails.md | 2 +- docs/reference/build.md | 2 +- docs/reference/docker-compose.md | 2 +- docs/reference/index.md | 6 +-- docs/reference/kill.md | 2 +- docs/reference/overview.md | 2 +- docs/reference/port.md | 2 +- docs/reference/pull.md | 2 +- docs/reference/run.md | 6 +-- docs/reference/scale.md | 2 +- docs/wordpress.md | 6 +-- docs/yml.md | 5 +-- requirements-dev.txt | 8 ++-- requirements.txt | 2 +- setup.py | 7 +++- .../extends/nonexistent-path-base.yml | 2 +- .../extends/nonexistent-path-child.yml | 2 +- .../docker-compose.yaml | 2 +- tests/integration/cli_test.py | 9 +++-- tests/integration/legacy_test.py | 4 +- tests/integration/project_test.py | 4 +- tests/integration/resilience_test.py | 4 +- tests/integration/service_test.py | 36 ++++++++--------- tests/integration/state_test.py | 12 +++--- tests/integration/testcases.py | 9 +++-- tests/unit/cli/docker_client_test.py | 5 ++- tests/unit/cli/verbose_proxy_test.py | 4 +- tests/unit/cli_test.py | 5 ++- tests/unit/config_test.py | 5 ++- tests/unit/container_test.py | 4 +- tests/unit/interpolation_test.py | 3 +- tests/unit/log_printer_test.py | 5 ++- tests/unit/progress_stream_test.py | 4 +- tests/unit/project_test.py | 13 +++--- tests/unit/service_test.py | 31 +++++++------- tests/unit/sort_service_test.py | 3 +- tests/unit/split_buffer_test.py | 5 ++- 64 files changed, 250 insertions(+), 223 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e725da61..4f18ddbf8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,7 +202,7 @@ The highlights: - There is a new `fig restart` command which restarts a service's containers. - Fig creates multiple containers in service by appending a number to the service name (e.g. `db_1`, `db_2`, etc). As a convenience, Fig will now give the first container an alias of the service name (e.g. `db`). - + This link alias is also a valid hostname and added to `/etc/hosts` so you can connect to linked services using their hostname. For example, instead of resolving the environment variables `DB_PORT_5432_TCP_ADDR` and `DB_PORT_5432_TCP_PORT`, you could just use the hostname `db` and port `5432` directly. - Volume definitions now support `ro` mode, expanding `~` and expanding environment variables. @@ -250,7 +250,7 @@ Thanks @ryanbrainard and @d11wtq! ------------------ - Fig now starts links when you run `fig run` or `fig up`. - + For example, if you have a `web` service which depends on a `db` service, `fig run web ...` will start the `db` service. - Environment variables can now be resolved from the environment that Fig is running in. Just specify it as a blank variable in your `fig.yml` and, if set, it'll be resolved: @@ -410,5 +410,3 @@ Big thanks to @tomstuart, @EnTeQuAk, @schickling, @aronasorman and @GeoffreyPlit ------------------ Initial release. - - diff --git a/README.md b/README.md index 7121f6a2d9..69423111e5 100644 --- a/README.md +++ b/README.md @@ -54,4 +54,4 @@ Want to help build Compose? Check out our [contributing documentation](https://g Releasing --------- -Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/RELEASE_PROCESS.md). \ No newline at end of file +Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/RELEASE_PROCESS.md). diff --git a/compose/cli/command.py b/compose/cli/command.py index 204ed52710..67176df271 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -1,20 +1,25 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from requests.exceptions import ConnectionError, SSLError +from __future__ import unicode_literals + import logging import os import re -import six +import six +from requests.exceptions import ConnectionError +from requests.exceptions import SSLError + +from . import errors +from . import verbose_proxy +from .. import __version__ from .. import config from ..project import Project from ..service import ConfigError -from .docopt_command import DocoptCommand -from .utils import call_silently, is_mac, is_ubuntu from .docker_client import docker_client -from . import verbose_proxy -from . import errors -from .. import __version__ +from .docopt_command import DocoptCommand +from .utils import call_silently +from .utils import is_mac +from .utils import is_ubuntu log = logging.getLogger(__name__) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 244bcbef2f..ad67d5639f 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -1,7 +1,8 @@ +import os +import ssl + from docker import Client from docker import tls -import ssl -import os def docker_client(): diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index 6eeb33a317..27f4b2bd7f 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -1,9 +1,11 @@ -from __future__ import unicode_literals from __future__ import absolute_import -import sys +from __future__ import unicode_literals +import sys from inspect import getdoc -from docopt import docopt, DocoptExit + +from docopt import docopt +from docopt import DocoptExit def docopt_full_help(docstring, *args, **kwargs): diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 135710d434..0569c1a0dd 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + from textwrap import dedent diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index b5b0b3c03d..9ed52c4aa5 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -1,6 +1,8 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals + import os + import texttable diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 9c5d35e187..ef484ca6cb 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -1,11 +1,11 @@ -from __future__ import unicode_literals from __future__ import absolute_import -import sys +from __future__ import unicode_literals +import sys from itertools import cycle -from .multiplexer import Multiplexer from . import colors +from .multiplexer import Multiplexer from .utils import split_buffer diff --git a/compose/cli/main.py b/compose/cli/main.py index b95a09c809..890a3c3717 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1,28 +1,32 @@ from __future__ import print_function from __future__ import unicode_literals -from inspect import getdoc -from operator import attrgetter + import logging import re import signal import sys +from inspect import getdoc +from operator import attrgetter -from docker.errors import APIError import dockerpty +from docker.errors import APIError from .. import __version__ from .. import legacy -from ..const import DEFAULT_TIMEOUT -from ..project import NoSuchService, ConfigurationError -from ..service import BuildError, NeedsBuildError from ..config import parse_environment +from ..const import DEFAULT_TIMEOUT from ..progress_stream import StreamOutputError +from ..project import ConfigurationError +from ..project import NoSuchService +from ..service import BuildError +from ..service import NeedsBuildError from .command import Command from .docopt_command import NoSuchCommand from .errors import UserError from .formatter import Formatter from .log_printer import LogPrinter -from .utils import yesno, get_version_info +from .utils import get_version_info +from .utils import yesno log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 955af63221..b502c351b7 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + from threading import Thread try: diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 7f2ba2e0dd..1bb497cd80 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -1,14 +1,16 @@ -from __future__ import unicode_literals from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals -from .. import __version__ import datetime -from docker import version as docker_py_version import os import platform -import subprocess import ssl +import subprocess + +from docker import version as docker_py_version + +from .. import __version__ def yesno(prompt, default=None): diff --git a/compose/cli/verbose_proxy.py b/compose/cli/verbose_proxy.py index a548983e1c..68dfabe521 100644 --- a/compose/cli/verbose_proxy.py +++ b/compose/cli/verbose_proxy.py @@ -1,8 +1,7 @@ - import functools -from itertools import chain import logging import pprint +from itertools import chain import six diff --git a/compose/config/__init__.py b/compose/config/__init__.py index 3907e5b67e..de6f10c949 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -1,10 +1,9 @@ -from .config import ( - DOCKER_CONFIG_KEYS, - ConfigDetails, - ConfigurationError, - find, - load, - parse_environment, - merge_environment, - get_service_name_from_net, -) # flake8: noqa +# flake8: noqa +from .config import ConfigDetails +from .config import ConfigurationError +from .config import DOCKER_CONFIG_KEYS +from .config import find +from .config import get_service_name_from_net +from .config import load +from .config import merge_environment +from .config import parse_environment diff --git a/compose/config/config.py b/compose/config/config.py index b79ef254df..ea122bc422 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,23 +1,19 @@ import logging import os import sys -import yaml from collections import namedtuple + import six +import yaml -from compose.cli.utils import find_candidates_in_parent_dirs - +from .errors import CircularReference +from .errors import ComposeFileNotFound +from .errors import ConfigurationError from .interpolation import interpolate_environment_variables -from .errors import ( - ConfigurationError, - CircularReference, - ComposeFileNotFound, -) -from .validation import ( - validate_against_schema, - validate_service_names, - validate_top_level_object -) +from .validation import validate_against_schema +from .validation import validate_service_names +from .validation import validate_top_level_object +from compose.cli.utils import find_candidates_in_parent_dirs DOCKER_CONFIG_KEYS = [ diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 8ebcc87596..f870ab4b27 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -1,11 +1,10 @@ +import logging import os from string import Template import six from .errors import ConfigurationError - -import logging log = logging.getLogger(__name__) diff --git a/compose/config/validation.py b/compose/config/validation.py index 26f3ca8ec7..8911f5ae1d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,9 +1,11 @@ -from functools import wraps +import json import os +from functools import wraps from docker.utils.ports import split_port -import json -from jsonschema import Draft4Validator, FormatChecker, ValidationError +from jsonschema import Draft4Validator +from jsonschema import FormatChecker +from jsonschema import ValidationError from .errors import ConfigurationError diff --git a/compose/container.py b/compose/container.py index 37ed1fe593..f727c8673a 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,10 +1,12 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals -import six from functools import reduce -from .const import LABEL_CONTAINER_NUMBER, LABEL_SERVICE +import six + +from .const import LABEL_CONTAINER_NUMBER +from .const import LABEL_SERVICE class Container(object): diff --git a/compose/legacy.py b/compose/legacy.py index 6fbf74d692..e8f4f95739 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -2,7 +2,8 @@ import logging import re from .const import LABEL_VERSION -from .container import get_container_name, Container +from .container import Container +from .container import get_container_name log = logging.getLogger(__name__) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 317c6e8157..1ccdb861bd 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,6 +1,6 @@ +import codecs import json import os -import codecs class StreamOutputError(Exception): diff --git a/compose/project.py b/compose/project.py index 276afb543f..eb395297c6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -1,12 +1,17 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from functools import reduce +from __future__ import unicode_literals + import logging +from functools import reduce from docker.errors import APIError -from .config import get_service_name_from_net, ConfigurationError -from .const import DEFAULT_TIMEOUT, LABEL_PROJECT, LABEL_SERVICE, LABEL_ONE_OFF +from .config import ConfigurationError +from .config import get_service_name_from_net +from .const import DEFAULT_TIMEOUT +from .const import LABEL_ONE_OFF +from .const import LABEL_PROJECT +from .const import LABEL_SERVICE from .container import Container from .legacy import check_for_legacy_containers from .service import Service diff --git a/compose/service.py b/compose/service.py index 7df5618c55..05e546c436 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1,33 +1,37 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from collections import namedtuple +from __future__ import unicode_literals + import logging -import re import os +import re import sys +from collections import namedtuple from operator import attrgetter import six from docker.errors import APIError -from docker.utils import create_host_config, LogConfig -from docker.utils.ports import build_port_bindings, split_port +from docker.utils import create_host_config +from docker.utils import LogConfig +from docker.utils.ports import build_port_bindings +from docker.utils.ports import split_port from . import __version__ -from .config import DOCKER_CONFIG_KEYS, merge_environment -from .const import ( - DEFAULT_TIMEOUT, - LABEL_CONTAINER_NUMBER, - LABEL_ONE_OFF, - LABEL_PROJECT, - LABEL_SERVICE, - LABEL_VERSION, - LABEL_CONFIG_HASH, -) +from .config import DOCKER_CONFIG_KEYS +from .config import merge_environment +from .config.validation import VALID_NAME_CHARS +from .const import DEFAULT_TIMEOUT +from .const import LABEL_CONFIG_HASH +from .const import LABEL_CONTAINER_NUMBER +from .const import LABEL_ONE_OFF +from .const import LABEL_PROJECT +from .const import LABEL_SERVICE +from .const import LABEL_VERSION from .container import Container from .legacy import check_for_legacy_containers -from .progress_stream import stream_output, StreamOutputError -from .utils import json_hash, parallel_execute -from .config.validation import VALID_NAME_CHARS +from .progress_stream import stream_output +from .progress_stream import StreamOutputError +from .utils import json_hash +from .utils import parallel_execute log = logging.getLogger(__name__) diff --git a/compose/utils.py b/compose/utils.py index 61d6d80243..bd8922670e 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -3,10 +3,11 @@ import hashlib import json import logging import sys +from Queue import Empty +from Queue import Queue +from threading import Thread from docker.errors import APIError -from Queue import Queue, Empty -from threading import Thread log = logging.getLogger(__name__) diff --git a/docs/README.md b/docs/README.md index 4d6465637f..8fbad30c58 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # Contributing to the Docker Compose documentation -The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. +The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. -You don't need to be a Hugo expert to contribute to the compose documentation. If you are familiar with Markdown, you can modify the content in the `docs` files. +You don't need to be a Hugo expert to contribute to the compose documentation. If you are familiar with Markdown, you can modify the content in the `docs` files. If you want to add a new file or change the location of the document in the menu, you do need to know a little more. @@ -23,7 +23,7 @@ If you want to add a new file or change the location of the document in the menu docker run --rm -it -e AWS_S3_BUCKET -e NOCACHE -p 8000:8000 -e DOCKERHOST "docs-base:test-tooling" hugo server --port=8000 --baseUrl=192.168.59.103 --bind=0.0.0.0 ERROR: 2015/06/13 MenuEntry's .Url is deprecated and will be removed in Hugo 0.15. Use .URL instead. 0 of 4 drafts rendered - 0 future content + 0 future content 12 pages created 0 paginator pages created 0 tags created @@ -52,7 +52,7 @@ The top of each Docker Compose documentation file contains TOML metadata. The me parent="smn_workw_compose" weight=2 +++ - + The metadata alone has this structure: @@ -64,7 +64,7 @@ The metadata alone has this structure: parent="smn_workw_compose" weight=2 +++ - + The `[menu.main]` section refers to navigation defined [in the main Docker menu](https://github.com/docker/docs-base/blob/hugo/config.toml). This metadata says *add a menu item called* Extending services in Compose *to the menu with the* `smn_workdw_compose` *identifier*. If you locate the menu in the configuration, you'll find *Create multi-container applications* is the menu title. You can move an article in the tree by specifying a new parent. You can shift the location of the item by changing its weight. Higher numbers are heavier and shift the item to the bottom of menu. Low or no numbers shift it up. @@ -73,5 +73,5 @@ You can move an article in the tree by specifying a new parent. You can shift th ## Other key documentation repositories The `docker/docs-base` repository contains [the Hugo theme and menu configuration](https://github.com/docker/docs-base). If you open the `Dockerfile` you'll see the `make docs` relies on this as a base image for building the Compose documentation. - + The `docker/docs.docker.com` repository contains [build system for building the Docker documentation site](https://github.com/docker/docs.docker.com). Fork this repository to build the entire documentation site. diff --git a/docs/index.md b/docs/index.md index 872b015881..4342b3686d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -161,7 +161,7 @@ Now, when you run `docker-compose up`, Compose will pull a Redis image, build an web_1 | * Running on http://0.0.0.0:5000/ web_1 | * Restarting with stat -If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. +If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. If you're not using Boot2docker and are on linux, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try localhost:5000. diff --git a/docs/install.md b/docs/install.md index d71aa0800d..7a2763edc0 100644 --- a/docs/install.md +++ b/docs/install.md @@ -14,7 +14,7 @@ weight=4 You can run Compose on OS X and 64-bit Linux. It is currently not supported on the Windows operating system. To install Compose, you'll need to install Docker -first. +first. Depending on how your system is configured, you may require `sudo` access to install Compose. If your system requires `sudo`, you will receive "Permission @@ -26,13 +26,13 @@ To install Compose, do the following: 1. Install Docker Engine version 1.7.1 or greater: * Mac OS X installation (installs both Engine and Compose) - + * Ubuntu installation - + * other system installations - + 2. Mac OS X users are done installing. Others should continue to the next step. - + 3. Go to the repository release page. 4. Enter the `curl` command in your termial. @@ -40,9 +40,9 @@ To install Compose, do the following: The command has the following format: curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - + If you have problems installing with `curl`, you can use `pip` instead: `pip install -U docker-compose` - + 4. Apply executable permissions to the binary: $ chmod +x /usr/local/bin/docker-compose @@ -85,7 +85,7 @@ To uninstall Docker Compose if you installed using `curl`: To uninstall Docker Compose if you installed using `pip`: $ pip uninstall docker-compose - + >**Note**: If you get a "Permission denied" error using either of the above >methods, you probably do not have the proper permissions to remove >`docker-compose`. To force the removal, prepend `sudo` to either of the above diff --git a/docs/pre-process.sh b/docs/pre-process.sh index 75e9611f2f..f1f6b7fec6 100755 --- a/docs/pre-process.sh +++ b/docs/pre-process.sh @@ -13,7 +13,7 @@ content_dir=(`ls -d /docs/content/*`) # 5 Change ](word) to ](/project/word) # 6 Change ](../../ to ](/project/ # 7 Change ](../ to ](/project/word) -# +# for i in "${content_dir[@]}" do : @@ -51,11 +51,10 @@ done for i in "${docker_dir[@]}" do : - if [ -d $i ] + if [ -d $i ] then - mv $i /docs/content/ + mv $i /docs/content/ fi done rm -rf /docs/content/docker - diff --git a/docs/production.md b/docs/production.md index 6005113695..3020a0c402 100644 --- a/docs/production.md +++ b/docs/production.md @@ -93,4 +93,3 @@ guide. - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) - diff --git a/docs/rails.md b/docs/rails.md index b73be90cb5..186f9b2bf2 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -117,7 +117,7 @@ Finally, you need to create the database. In another terminal, run: $ docker-compose run web rake db:create -That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. +That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. ## More Compose documentation diff --git a/docs/reference/build.md b/docs/reference/build.md index b6e27bb264..77d87def49 100644 --- a/docs/reference/build.md +++ b/docs/reference/build.md @@ -20,4 +20,4 @@ Options: Services are built once and then tagged as `project_service`, e.g., `composetest_db`. If you change a service's Dockerfile or the contents of its -build directory, run `docker-compose build` to rebuild it. \ No newline at end of file +build directory, run `docker-compose build` to rebuild it. diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 46afba13c1..6c46b31d18 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -5,7 +5,7 @@ description = "docker-compose Command Binary" keywords = ["fig, composition, compose, docker, orchestration, cli, docker-compose"] [menu.main] parent = "smn_compose_cli" -weight=-2 +weight=-2 +++ diff --git a/docs/reference/index.md b/docs/reference/index.md index 5651e5bf05..e7a07b09aa 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -5,7 +5,7 @@ description = "Compose CLI reference" keywords = ["fig, composition, compose, docker, orchestration, cli, reference"] [menu.main] identifier = "smn_compose_cli" -parent = "smn_compose_ref" +parent = "smn_compose_ref" +++ @@ -15,7 +15,7 @@ The following pages describe the usage information for the [docker-compose](/ref * [build](/reference/build.md) * [help](/reference/help.md) -* [kill](/reference/kill.md) +* [kill](/reference/kill.md) * [ps](/reference/ps.md) * [restart](/reference/restart.md) * [run](/reference/run.md) @@ -23,7 +23,7 @@ The following pages describe the usage information for the [docker-compose](/ref * [up](/reference/up.md) * [logs](/reference/logs.md) * [port](/reference/port.md) -* [pull](/reference/pull.md) +* [pull](/reference/pull.md) * [rm](/reference/rm.md) * [scale](/reference/scale.md) * [stop](/reference/stop.md) diff --git a/docs/reference/kill.md b/docs/reference/kill.md index e5dd057361..dc4bf23a1b 100644 --- a/docs/reference/kill.md +++ b/docs/reference/kill.md @@ -21,4 +21,4 @@ Options: Forces running containers to stop by sending a `SIGKILL` signal. Optionally the signal can be passed, for example: - $ docker-compose kill -s SIGINT \ No newline at end of file + $ docker-compose kill -s SIGINT diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 458dea4046..7425aa5e8a 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -5,7 +5,7 @@ description = "Introduction to the CLI" keywords = ["fig, composition, compose, docker, orchestration, cli, reference"] [menu.main] parent = "smn_compose_cli" -weight=-2 +weight=-2 +++ diff --git a/docs/reference/port.md b/docs/reference/port.md index 76f93f2393..c946a97d39 100644 --- a/docs/reference/port.md +++ b/docs/reference/port.md @@ -20,4 +20,4 @@ Options: instances of a service [default: 1] ``` -Prints the public port for a port binding. \ No newline at end of file +Prints the public port for a port binding. diff --git a/docs/reference/pull.md b/docs/reference/pull.md index e5b5d166ff..d655dd93be 100644 --- a/docs/reference/pull.md +++ b/docs/reference/pull.md @@ -15,4 +15,4 @@ parent = "smn_compose_cli" Usage: pull [options] [SERVICE...] ``` -Pulls service images. \ No newline at end of file +Pulls service images. diff --git a/docs/reference/run.md b/docs/reference/run.md index 93ae0212b2..c1efb9a773 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -27,7 +27,7 @@ Options: -T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. ``` -Runs a one-time command against a service. For example, the following command starts the `web` service and runs `bash` as its command. +Runs a one-time command against a service. For example, the following command starts the `web` service and runs `bash` as its command. $ docker-compose run web bash @@ -52,7 +52,3 @@ This would open up an interactive PostgreSQL shell for the linked `db` container If you do not want the `run` command to start linked containers, specify the `--no-deps` flag: $ docker-compose run --no-deps web python manage.py shell - - - - diff --git a/docs/reference/scale.md b/docs/reference/scale.md index 9541830097..75140ee9e5 100644 --- a/docs/reference/scale.md +++ b/docs/reference/scale.md @@ -18,4 +18,4 @@ Sets the number of containers to run for a service. Numbers are specified as arguments in the form `service=num`. For example: - $ docker-compose scale web=2 worker=3 \ No newline at end of file + $ docker-compose scale web=2 worker=3 diff --git a/docs/wordpress.md b/docs/wordpress.md index 8440fdbb41..ab22e2a0df 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -13,7 +13,7 @@ weight=6 # Quickstart Guide: Compose and Wordpress You can use Compose to easily run Wordpress in an isolated environment built -with Docker containers. +with Docker containers. ## Define the project @@ -36,7 +36,7 @@ your Dockerfile should be: ADD . /code This tells Docker how to build an image defining a container that contains PHP -and Wordpress. +and Wordpress. Next you'll create a `docker-compose.yml` file that will start your web service and a separate MySQL instance: @@ -108,7 +108,7 @@ Second, `router.php` tells PHP's built-in web server how to run Wordpress: With those four files in place, run `docker-compose up` inside your Wordpress directory and it'll pull and build the needed images, and then start the web and -database containers. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. +database containers. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. ## More Compose documentation diff --git a/docs/yml.md b/docs/yml.md index 9662208641..6fb31a7db9 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -19,7 +19,7 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. -Values for configuration options can contain environment variables, e.g. +Values for configuration options can contain environment variables, e.g. `image: postgres:${POSTGRES_VERSION}`. For more details, see the section on [variable substitution](#variable-substitution). @@ -353,7 +353,7 @@ Custom DNS search domains. Can be a single value or a list. ### devices -List of device mappings. Uses the same format as the `--device` docker +List of device mappings. Uses the same format as the `--device` docker client create option. devices: @@ -433,4 +433,3 @@ dollar sign (`$$`). - [Command line reference](/reference) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) - diff --git a/requirements-dev.txt b/requirements-dev.txt index c5d9c10645..97fc4fed86 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ +coverage==3.7.1 +flake8==2.3.0 +git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller mock >= 1.0.1 nose==1.3.4 -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller -unittest2==0.8.0 -flake8==2.3.0 pep8==1.6.1 -coverage==3.7.1 +unittest2==0.8.0 diff --git a/requirements.txt b/requirements.txt index 6416876861..e93db7b361 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ PyYAML==3.10 -jsonschema==2.5.1 docker-py==1.3.1 dockerpty==0.3.4 docopt==0.6.1 +jsonschema==2.5.1 requests==2.6.1 six==1.7.3 texttable==0.8.2 diff --git a/setup.py b/setup.py index 1f9c981d1b..2f6dad7a9b 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import unicode_literals from __future__ import absolute_import -from setuptools import setup, find_packages +from __future__ import unicode_literals + import codecs import os import re import sys +from setuptools import find_packages +from setuptools import setup + def read(*parts): path = os.path.join(os.path.dirname(__file__), *parts) diff --git a/tests/fixtures/extends/nonexistent-path-base.yml b/tests/fixtures/extends/nonexistent-path-base.yml index 1cf9a304ae..4e6c82b0d7 100644 --- a/tests/fixtures/extends/nonexistent-path-base.yml +++ b/tests/fixtures/extends/nonexistent-path-base.yml @@ -3,4 +3,4 @@ dnebase: command: /bin/true environment: - FOO=1 - - BAR=1 \ No newline at end of file + - BAR=1 diff --git a/tests/fixtures/extends/nonexistent-path-child.yml b/tests/fixtures/extends/nonexistent-path-child.yml index aab11459b1..d3b732f2a3 100644 --- a/tests/fixtures/extends/nonexistent-path-child.yml +++ b/tests/fixtures/extends/nonexistent-path-child.yml @@ -5,4 +5,4 @@ dnechild: image: busybox command: /bin/true environment: - - BAR=2 \ No newline at end of file + - BAR=2 diff --git a/tests/fixtures/longer-filename-composefile/docker-compose.yaml b/tests/fixtures/longer-filename-composefile/docker-compose.yaml index b55a9e1245..a4eba2d05d 100644 --- a/tests/fixtures/longer-filename-composefile/docker-compose.yaml +++ b/tests/fixtures/longer-filename-composefile/docker-compose.yaml @@ -1,3 +1,3 @@ definedinyamlnotyml: image: busybox:latest - command: top \ No newline at end of file + command: top diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 38f8ee4648..8bdcadd529 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,15 +1,16 @@ from __future__ import absolute_import -from operator import attrgetter -import sys + import os import shlex +import sys +from operator import attrgetter -from six import StringIO from mock import patch +from six import StringIO from .testcases import DockerClientTestCase -from compose.cli.main import TopLevelCommand from compose.cli.errors import UserError +from compose.cli.main import TopLevelCommand from compose.project import NoSuchService diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 9913bbb0fe..fa983e6d59 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,11 +1,11 @@ import unittest -from mock import Mock from docker.errors import APIError +from mock import Mock +from .testcases import DockerClientTestCase from compose import legacy from compose.project import Project -from .testcases import DockerClientTestCase class UtilitiesTestCase(unittest.TestCase): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ad2fe4fea1..51619cb5ec 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,10 +1,10 @@ from __future__ import unicode_literals +from .testcases import DockerClientTestCase from compose import config from compose.const import LABEL_PROJECT -from compose.project import Project from compose.container import Container -from .testcases import DockerClientTestCase +from compose.project import Project def build_service_dicts(service_config): diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index e0c76f299d..b1faf99dff 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -1,10 +1,10 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals import mock -from compose.project import Project from .testcases import DockerClientTestCase +from compose.project import Project class ResilienceTest(DockerClientTestCase): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 050a3bf622..1d53465f63 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1,30 +1,28 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals + import os +import shutil +import tempfile from os import path from docker.errors import APIError from mock import patch -import tempfile -import shutil -from six import StringIO, text_type +from six import StringIO +from six import text_type -from compose import __version__ -from compose.const import ( - LABEL_CONTAINER_NUMBER, - LABEL_ONE_OFF, - LABEL_PROJECT, - LABEL_SERVICE, - LABEL_VERSION, -) -from compose.service import ( - ConfigError, - ConvergencePlan, - Service, - build_extra_hosts, -) -from compose.container import Container from .testcases import DockerClientTestCase +from compose import __version__ +from compose.const import LABEL_CONTAINER_NUMBER +from compose.const import LABEL_ONE_OFF +from compose.const import LABEL_PROJECT +from compose.const import LABEL_SERVICE +from compose.const import LABEL_VERSION +from compose.container import Container +from compose.service import build_extra_hosts +from compose.service import ConfigError +from compose.service import ConvergencePlan +from compose.service import Service def create_and_start_container(service, **override_options): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index b124b19ffc..3d4a5b5aa6 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -1,13 +1,13 @@ from __future__ import unicode_literals -import tempfile -import shutil -import os -from compose import config -from compose.project import Project -from compose.const import LABEL_CONFIG_HASH +import os +import shutil +import tempfile from .testcases import DockerClientTestCase +from compose import config +from compose.const import LABEL_CONFIG_HASH +from compose.project import Project class ProjectTestCase(DockerClientTestCase): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a7929088be..e239010ea2 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,11 +1,12 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from compose.service import Service +from __future__ import unicode_literals + +from .. import unittest +from compose.cli.docker_client import docker_client from compose.config.config import ServiceLoader from compose.const import LABEL_PROJECT -from compose.cli.docker_client import docker_client from compose.progress_stream import stream_output -from .. import unittest +from compose.service import Service class DockerClientTestCase(unittest.TestCase): diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 44bdbb291e..6c2dc5f81e 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -1,11 +1,12 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals + import os import mock -from tests import unittest from compose.cli import docker_client +from tests import unittest class DockerClientTestCase(unittest.TestCase): diff --git a/tests/unit/cli/verbose_proxy_test.py b/tests/unit/cli/verbose_proxy_test.py index 59417bb3ef..6036974c6f 100644 --- a/tests/unit/cli/verbose_proxy_test.py +++ b/tests/unit/cli/verbose_proxy_test.py @@ -1,8 +1,8 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from tests import unittest +from __future__ import unicode_literals from compose.cli import verbose_proxy +from tests import unittest class VerboseProxyTestCase(unittest.TestCase): diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index e11f6f14af..35be4e9264 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -1,11 +1,12 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals + import os -from .. import unittest import docker import mock +from .. import unittest from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index e61172562c..3d1a53214d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -1,9 +1,10 @@ -import mock import os import shutil import tempfile -from .. import unittest +import mock + +from .. import unittest from compose.config import config from compose.config.errors import ConfigurationError diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index c537a8cf55..e2381c7c27 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals -from .. import unittest -import mock import docker +import mock +from .. import unittest from compose.container import Container from compose.container import get_container_name diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py index fb95422b0e..7444884cb8 100644 --- a/tests/unit/interpolation_test.py +++ b/tests/unit/interpolation_test.py @@ -1,7 +1,8 @@ import unittest -from compose.config.interpolation import interpolate, InvalidInterpolation from compose.config.interpolation import BlankDefaultDict as bddict +from compose.config.interpolation import interpolate +from compose.config.interpolation import InvalidInterpolation class InterpolationTest(unittest.TestCase): diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py index e40a1f75da..bfd16affe1 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/log_printer_test.py @@ -1,9 +1,10 @@ -from __future__ import unicode_literals from __future__ import absolute_import +from __future__ import unicode_literals + import os -from compose.cli.log_printer import LogPrinter from .. import unittest +from compose.cli.log_printer import LogPrinter class LogPrinterTest(unittest.TestCase): diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 317b77e9f2..5674f4e4e8 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -1,10 +1,10 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from tests import unittest +from __future__ import unicode_literals from six import StringIO from compose import progress_stream +from tests import unittest class ProgressStreamTestCase(unittest.TestCase): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 93bf12ff57..7d633c9505 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,12 +1,13 @@ from __future__ import unicode_literals -from .. import unittest -from compose.service import Service -from compose.project import Project -from compose.container import Container -from compose.const import LABEL_SERVICE -import mock import docker +import mock + +from .. import unittest +from compose.const import LABEL_SERVICE +from compose.container import Container +from compose.project import Project +from compose.service import Service class ProjectTest(unittest.TestCase): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 2965d6c891..12bb4ac2d7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1,25 +1,24 @@ -from __future__ import unicode_literals from __future__ import absolute_import - -from .. import unittest -import mock +from __future__ import unicode_literals import docker +import mock from docker.utils import LogConfig -from compose.service import Service +from .. import unittest +from compose.const import LABEL_ONE_OFF +from compose.const import LABEL_PROJECT +from compose.const import LABEL_SERVICE from compose.container import Container -from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF -from compose.service import ( - ConfigError, - NeedsBuildError, - NoSuchImageError, - build_volume_binding, - get_container_data_volumes, - merge_volume_bindings, - parse_repository_tag, - parse_volume_spec, -) +from compose.service import build_volume_binding +from compose.service import ConfigError +from compose.service import get_container_data_volumes +from compose.service import merge_volume_bindings +from compose.service import NeedsBuildError +from compose.service import NoSuchImageError +from compose.service import parse_repository_tag +from compose.service import parse_volume_spec +from compose.service import Service class ServiceTest(unittest.TestCase): diff --git a/tests/unit/sort_service_test.py b/tests/unit/sort_service_test.py index f42a947484..a7e522a1dd 100644 --- a/tests/unit/sort_service_test.py +++ b/tests/unit/sort_service_test.py @@ -1,5 +1,6 @@ -from compose.project import sort_service_dicts, DependencyError from .. import unittest +from compose.project import DependencyError +from compose.project import sort_service_dicts class SortServiceTest(unittest.TestCase): diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index 8eb54177aa..efd99411ae 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -1,7 +1,8 @@ -from __future__ import unicode_literals from __future__ import absolute_import -from compose.cli.utils import split_buffer +from __future__ import unicode_literals + from .. import unittest +from compose.cli.utils import split_buffer class SplitBufferTest(unittest.TestCase): From 809443d6d03e1ec687c01e546ddd9031b56ce40c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Aug 2014 17:36:46 -0400 Subject: [PATCH 074/337] Support python 3 Signed-off-by: Daniel Nephin --- Dockerfile | 4 +- MANIFEST.in | 2 +- compose/cli/log_printer.py | 6 ++- compose/cli/utils.py | 3 +- compose/progress_stream.py | 9 ++-- compose/project.py | 1 + compose/service.py | 2 +- ...ements-dev.txt => requirements-dev-py2.txt | 0 requirements-dev-py3.txt | 2 + setup.py | 5 +- tests/__init__.py | 5 ++ tests/integration/cli_test.py | 46 +++++++++---------- tests/integration/service_test.py | 24 +++++----- tests/unit/cli/docker_client_test.py | 3 +- tests/unit/cli/verbose_proxy_test.py | 7 ++- tests/unit/cli_test.py | 2 +- tests/unit/config_test.py | 18 ++++---- tests/unit/container_test.py | 2 +- tests/unit/log_printer_test.py | 9 ++-- tests/unit/project_test.py | 2 +- tests/unit/service_test.py | 2 +- tests/unit/split_buffer_test.py | 36 +++++++-------- tox.ini | 23 +++++++++- 23 files changed, 128 insertions(+), 85 deletions(-) rename requirements-dev.txt => requirements-dev-py2.txt (100%) create mode 100644 requirements-dev-py3.txt diff --git a/Dockerfile b/Dockerfile index a4cc99fea0..1986ac5a5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,8 @@ WORKDIR /code/ ADD requirements.txt /code/ RUN pip install -r requirements.txt -ADD requirements-dev.txt /code/ -RUN pip install -r requirements-dev.txt +ADD requirements-dev-py2.txt /code/ +RUN pip install -r requirements-dev-py2.txt RUN pip install tox==2.1.1 diff --git a/MANIFEST.in b/MANIFEST.in index 7d48d347a8..7420485961 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include Dockerfile include LICENSE include requirements.txt -include requirements-dev.txt +include requirements-dev*.txt include tox.ini include *.md include compose/config/schema.json diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index ef484ca6cb..c7d0b638f8 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals import sys from itertools import cycle +import six + from . import colors from .multiplexer import Multiplexer from .utils import split_buffer @@ -20,6 +22,8 @@ class LogPrinter(object): def run(self): mux = Multiplexer(self.generators) for line in mux.loop(): + if isinstance(line, six.text_type) and not six.PY3: + line = line.encode('utf-8') self.output.write(line) def _calculate_prefix_width(self, containers): @@ -52,7 +56,7 @@ class LogPrinter(object): return generators def _make_log_generator(self, container, color_fn): - prefix = color_fn(self._generate_prefix(container)).encode('utf-8') + prefix = color_fn(self._generate_prefix(container)) # Attach to container before log printer starts running line_generator = split_buffer(self._attach(container), '\n') diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 1bb497cd80..b6c83f9e1f 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -9,6 +9,7 @@ import ssl import subprocess from docker import version as docker_py_version +from six.moves import input from .. import __version__ @@ -23,7 +24,7 @@ def yesno(prompt, default=None): Unrecognised input (anything other than "y", "n", "yes", "no" or "") will return None. """ - answer = raw_input(prompt).strip().lower() + answer = input(prompt).strip().lower() if answer == "y" or answer == "yes": return True diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 1ccdb861bd..582c09fb9e 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,6 +1,7 @@ import codecs import json -import os + +import six class StreamOutputError(Exception): @@ -8,8 +9,9 @@ class StreamOutputError(Exception): def stream_output(output, stream): - is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) - stream = codecs.getwriter('utf-8')(stream) + is_terminal = hasattr(stream, 'isatty') and stream.isatty() + if not six.PY3: + stream = codecs.getwriter('utf-8')(stream) all_events = [] lines = {} diff = 0 @@ -55,7 +57,6 @@ def print_output_event(event, stream, is_terminal): # erase current line stream.write("%c[2K\r" % 27) terminator = "\r" - pass elif 'progressDetail' in event: return diff --git a/compose/project.py b/compose/project.py index eb395297c6..d14941e72b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -17,6 +17,7 @@ from .legacy import check_for_legacy_containers from .service import Service from .utils import parallel_execute + log = logging.getLogger(__name__) diff --git a/compose/service.py b/compose/service.py index 05e546c436..647516ba84 100644 --- a/compose/service.py +++ b/compose/service.py @@ -724,7 +724,7 @@ class Service(object): try: all_events = stream_output(build_output, sys.stdout) except StreamOutputError as e: - raise BuildError(self, unicode(e)) + raise BuildError(self, six.text_type(e)) # Ensure the HTTP connection is not reused for another # streaming command, as the Docker daemon can sometimes diff --git a/requirements-dev.txt b/requirements-dev-py2.txt similarity index 100% rename from requirements-dev.txt rename to requirements-dev-py2.txt diff --git a/requirements-dev-py3.txt b/requirements-dev-py3.txt new file mode 100644 index 0000000000..a2ba1c8b45 --- /dev/null +++ b/requirements-dev-py3.txt @@ -0,0 +1,2 @@ +flake8 +nose >= 1.3.0 diff --git a/setup.py b/setup.py index 2f6dad7a9b..b7fd440348 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,11 @@ tests_require = [ ] -if sys.version_info < (2, 7): +if sys.version_info < (2, 6): tests_require.append('unittest2') +if sys.version_info[:1] < (3,): + tests_require.append('pyinstaller') + tests_require.append('mock >= 1.0.1') setup( diff --git a/tests/__init__.py b/tests/__init__.py index 08a7865e90..d3cfb86491 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,3 +4,8 @@ if sys.version_info >= (2, 7): import unittest # NOQA else: import unittest2 as unittest # NOQA + +try: + from unittest import mock +except ImportError: + import mock # NOQA diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 8bdcadd529..609370a3e1 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -5,9 +5,9 @@ import shlex import sys from operator import attrgetter -from mock import patch from six import StringIO +from .. import mock from .testcases import DockerClientTestCase from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand @@ -51,13 +51,13 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = old_base_dir # TODO: address the "Inappropriate ioctl for device" warnings in test output - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps(self, mock_stdout): self.project.get_service('simple').create_container() self.command.dispatch(['ps'], None) self.assertIn('simplecomposefile_simple_1', mock_stdout.getvalue()) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps_default_composefile(self, mock_stdout): self.command.base_dir = 'tests/fixtures/multiple-composefiles' self.command.dispatch(['up', '-d'], None) @@ -68,7 +68,7 @@ class CLITestCase(DockerClientTestCase): self.assertIn('multiplecomposefiles_another_1', output) self.assertNotIn('multiplecomposefiles_yetanother_1', output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps_alternate_composefile(self, mock_stdout): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') @@ -83,19 +83,19 @@ class CLITestCase(DockerClientTestCase): self.assertNotIn('multiplecomposefiles_another_1', output) self.assertIn('multiplecomposefiles_yetanother_1', output) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_pull(self, mock_logging): self.command.dispatch(['pull'], None) mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') mock_logging.info.assert_any_call('Pulling another (busybox:latest)...') - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_pull_with_digest(self, mock_logging): self.command.dispatch(['-f', 'digest.yml', 'pull'], None) mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') mock_logging.info.assert_any_call('Pulling digest (busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_build_no_cache(self, mock_stdout): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) @@ -189,7 +189,7 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(config['AttachStdout']) self.assertFalse(config['AttachStdin']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_without_links(self, mock_stdout): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', 'console', '/bin/true'], None) @@ -202,7 +202,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(config['AttachStdout']) self.assertTrue(config['AttachStdin']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_links(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', 'web', '/bin/true'], None) @@ -211,14 +211,14 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_with_no_deps(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', '--no-deps', 'web', '/bin/true'], None) db = self.project.get_service('db') self.assertEqual(len(db.containers()), 0) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_does_not_recreate_linked_containers(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d', 'db'], None) @@ -234,7 +234,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_without_command(self, _): self.command.base_dir = 'tests/fixtures/commands-composefile' self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') @@ -255,7 +255,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_entrypoint_overridden(self, _): self.command.base_dir = 'tests/fixtures/dockerfile_with_entrypoint' name = 'service' @@ -270,7 +270,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/echo', u'helloworld'], ) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_user_overridden(self, _): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' @@ -281,7 +281,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_user_overridden_short_form(self, _): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' @@ -292,7 +292,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_environement_overridden(self, _): name = 'service' self.command.base_dir = 'tests/fixtures/environment-composefile' @@ -312,7 +312,7 @@ class CLITestCase(DockerClientTestCase): # make sure a value with a = don't crash out self.assertEqual('moto=bobo', container.environment['allo']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_without_map_ports(self, __): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' @@ -330,7 +330,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_random, None) self.assertEqual(port_assigned, None) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_map_ports(self, __): # create one off container @@ -353,7 +353,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_explicitly_maped_ports(self, __): # create one off container @@ -372,7 +372,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_explicitly_maped_ip_ports(self, __): # create one off container @@ -508,7 +508,7 @@ class CLITestCase(DockerClientTestCase): self.command.dispatch(['up', '-d'], None) container = self.project.get_service('simple').get_container() - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def get_port(number, mock_stdout): self.command.dispatch(['port', 'simple', str(number)], None) return mock_stdout.getvalue().rstrip() @@ -525,7 +525,7 @@ class CLITestCase(DockerClientTestCase): self.project.containers(service_names=['simple']), key=attrgetter('name')) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def get_port(number, mock_stdout, index=None): if index is None: self.command.dispatch(['port', 'simple', str(number)], None) @@ -547,7 +547,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) - @patch.dict(os.environ) + @mock.patch.dict(os.environ) def test_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' os.environ['HOME'] = '/tmp/home-dir' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 1d53465f63..fe54d4ae24 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -7,10 +7,10 @@ import tempfile from os import path from docker.errors import APIError -from mock import patch from six import StringIO from six import text_type +from .. import mock from .testcases import DockerClientTestCase from compose import __version__ from compose.const import LABEL_CONTAINER_NUMBER @@ -460,7 +460,7 @@ class ServiceTest(DockerClientTestCase): ) container = create_and_start_container(service) container.wait() - self.assertIn('success', container.logs()) + self.assertIn(b'success', container.logs()) self.assertEqual(len(self.client.images(name='composetest_test')), 1) def test_start_container_uses_tagged_image_if_it_exists(self): @@ -473,7 +473,7 @@ class ServiceTest(DockerClientTestCase): ) container = create_and_start_container(service) container.wait() - self.assertIn('success', container.logs()) + self.assertIn(b'success', container.logs()) def test_start_container_creates_ports(self): service = self.create_service('web', ports=[8000]) @@ -581,7 +581,7 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_stopped_containers(self, mock_stdout): """ Given there are some stopped containers and scale is called with a @@ -608,7 +608,7 @@ class ServiceTest(DockerClientTestCase): self.assertNotIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_stopped_containers_and_needing_creation(self, mock_stdout): """ Given there are some stopped containers and scale is called with a @@ -632,7 +632,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_api_returns_errors(self, mock_stdout): """ Test that when scaling if the API returns an error, that error is handled @@ -642,7 +642,7 @@ class ServiceTest(DockerClientTestCase): next_number = service._next_container_number() service.create_container(number=next_number, quiet=True) - with patch( + with mock.patch( 'compose.container.Container.create', side_effect=APIError(message="testing", response={}, explanation="Boom")): @@ -652,7 +652,7 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(service.containers()[0].is_running) self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_api_returns_unexpected_exception(self, mock_stdout): """ Test that when scaling if the API returns an error, that is not of type @@ -662,7 +662,7 @@ class ServiceTest(DockerClientTestCase): next_number = service._next_container_number() service.create_container(number=next_number, quiet=True) - with patch( + with mock.patch( 'compose.container.Container.create', side_effect=ValueError("BOOM")): with self.assertRaises(ValueError): @@ -671,7 +671,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_scale_with_desired_number_already_achieved(self, mock_log): """ Test that calling scale with a desired number that is equal to the @@ -694,7 +694,7 @@ class ServiceTest(DockerClientTestCase): captured_output = mock_log.info.call_args[0] self.assertIn('Desired container number already achieved', captured_output) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): """ Test that calling scale on a service that has a custom container name @@ -815,7 +815,7 @@ class ServiceTest(DockerClientTestCase): for k, v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): self.assertEqual(env[k], v) - @patch.dict(os.environ) + @mock.patch.dict(os.environ) def test_resolve_env(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 6c2dc5f81e..5ccde73ad3 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -3,9 +3,8 @@ from __future__ import unicode_literals import os -import mock - from compose.cli import docker_client +from tests import mock from tests import unittest diff --git a/tests/unit/cli/verbose_proxy_test.py b/tests/unit/cli/verbose_proxy_test.py index 6036974c6f..f77568dc08 100644 --- a/tests/unit/cli/verbose_proxy_test.py +++ b/tests/unit/cli/verbose_proxy_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import six + from compose.cli import verbose_proxy from tests import unittest @@ -8,7 +10,8 @@ from tests import unittest class VerboseProxyTestCase(unittest.TestCase): def test_format_call(self): - expected = "(u'arg1', True, key=u'value')" + prefix = '' if six.PY3 else 'u' + expected = "(%(p)s'arg1', True, key=%(p)s'value')" % dict(p=prefix) actual = verbose_proxy.format_call( ("arg1", True), {'key': 'value'}) @@ -21,7 +24,7 @@ class VerboseProxyTestCase(unittest.TestCase): self.assertEqual(expected, actual) def test_format_return(self): - expected = "{u'Id': u'ok'}" + expected = repr({'Id': 'ok'}) actual = verbose_proxy.format_return({'Id': 'ok'}, 2) self.assertEqual(expected, actual) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 35be4e9264..7d22ad02ff 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import os import docker -import mock +from .. import mock from .. import unittest from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 3d1a53214d..7ecb6c4a2d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -1,9 +1,11 @@ +from __future__ import print_function + import os import shutil import tempfile +from operator import itemgetter -import mock - +from .. import mock from .. import unittest from compose.config import config from compose.config.errors import ConfigurationError @@ -30,7 +32,7 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual( - sorted(service_dicts, key=lambda d: d['name']), + sorted(service_dicts, key=itemgetter('name')), sorted([ { 'name': 'bar', @@ -41,7 +43,7 @@ class ConfigTest(unittest.TestCase): 'name': 'foo', 'image': 'busybox', } - ], key=lambda d: d['name']) + ], key=itemgetter('name')) ) def test_load_throws_error_when_not_dict(self): @@ -885,24 +887,24 @@ class ExtendsTest(unittest.TestCase): other_config = {'web': {'links': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) with self.assertRaisesRegexp(ConfigurationError, 'volumes_from'): other_config = {'web': {'volumes_from': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) with self.assertRaisesRegexp(ConfigurationError, 'net'): other_config = {'web': {'net': 'container:db'}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) other_config = {'web': {'net': 'host'}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) def test_volume_path(self): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index e2381c7c27..1eba9f656d 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import docker -import mock +from .. import mock from .. import unittest from compose.container import Container from compose.container import get_container_name diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py index bfd16affe1..f3fa64c614 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/log_printer_test.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import os +import six + from .. import unittest from compose.cli.log_printer import LogPrinter @@ -30,16 +32,17 @@ class LogPrinterTest(unittest.TestCase): output = self.get_default_output() self.assertIn('\033[', output) + @unittest.skipIf(six.PY3, "Only test unicode in python2") def test_unicode(self): - glyph = u'\u2022'.encode('utf-8') + glyph = u'\u2022' def reader(*args, **kwargs): - yield glyph + b'\n' + yield glyph + '\n' container = MockContainer(reader) output = run_log_printer([container]) - self.assertIn(glyph, output) + self.assertIn(glyph, output.decode('utf-8')) def run_log_printer(containers, monochrome=False): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 7d633c9505..37ebe5148d 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import docker -import mock +from .. import mock from .. import unittest from compose.const import LABEL_SERVICE from compose.container import Container diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 12bb4ac2d7..3bb3e1722b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker -import mock from docker.utils import LogConfig +from .. import mock from .. import unittest from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index efd99411ae..1164609937 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -8,38 +8,38 @@ from compose.cli.utils import split_buffer class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): - yield b'abc\n' - yield b'def\n' - yield b'ghi\n' + yield 'abc\n' + yield 'def\n' + yield 'ghi\n' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi\n']) def test_no_end_separator(self): def reader(): - yield b'abc\n' - yield b'def\n' - yield b'ghi' + yield 'abc\n' + yield 'def\n' + yield 'ghi' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_multiple_line_chunk(self): def reader(): - yield b'abc\ndef\nghi' + yield 'abc\ndef\nghi' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_chunked_line(self): def reader(): - yield b'a' - yield b'b' - yield b'c' - yield b'\n' - yield b'd' + yield 'a' + yield 'b' + yield 'c' + yield '\n' + yield 'd' - self.assert_produces(reader, [b'abc\n', b'd']) + self.assert_produces(reader, ['abc\n', 'd']) def test_preserves_unicode_sequences_within_lines(self): - string = u"a\u2022c\n".encode('utf-8') + string = u"a\u2022c\n" def reader(): yield string @@ -47,7 +47,7 @@ class SplitBufferTest(unittest.TestCase): self.assert_produces(reader, [string]) def assert_produces(self, reader, expectations): - split = split_buffer(reader(), b'\n') + split = split_buffer(reader(), '\n') for (actual, expected) in zip(split, expectations): self.assertEqual(type(actual), type(expected)) diff --git a/tox.ini b/tox.ini index 3a69c5784c..35523a9697 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pre-commit +envlist = py27,py34,pre-commit [testenv] usedevelop=True @@ -7,7 +7,6 @@ passenv = LD_LIBRARY_PATH deps = -rrequirements.txt - -rrequirements-dev.txt commands = nosetests -v {posargs} flake8 compose tests setup.py @@ -20,6 +19,26 @@ commands = pre-commit install pre-commit run --all-files +[testenv:py26] +deps = + {[testenv]deps} + -rrequirements-dev-py2.txt + +[testenv:py27] +deps = {[testenv:py26]deps} + +[testenv:pypy] +deps = {[testenv:py26]deps} + +[testenv:py33] +deps = + {[testenv]deps} + -rrequirements-dev-py3.txt + +[testenv:py34] +deps = {[testenv:py33]deps} + + [flake8] # ignore line-length for now ignore = E501,E203 From 9aa61e596e2475fa0bbcf227f2c388f6a9df471a Mon Sep 17 00:00:00 2001 From: funkyfuture Date: Thu, 26 Mar 2015 23:28:02 +0100 Subject: [PATCH 075/337] Run tests against Python 2.6, 2.7, 3.3, 3.4 and PyPy2 In particular it includes: - some extension of CONTRIBUTING.md - one fix for Python 2.6 in tests/integration/cli_test.py - one fix for Python 3.3 in tests/integration/service_test.py - removal of unused imports Make stream_output Python 3-compatible Signed-off-by: Frank Sachsenheim --- Dockerfile | 4 ++-- compose/container.py | 7 +++---- compose/progress_stream.py | 2 ++ compose/project.py | 2 +- requirements-dev.txt | 2 ++ script/test-versions | 2 +- setup.py | 2 +- tests/integration/cli_test.py | 2 +- tests/integration/service_test.py | 2 +- tests/unit/progress_stream_test.py | 1 - tox.ini | 3 ++- 11 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 requirements-dev.txt diff --git a/Dockerfile b/Dockerfile index 1986ac5a5b..a4cc99fea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,8 @@ WORKDIR /code/ ADD requirements.txt /code/ RUN pip install -r requirements.txt -ADD requirements-dev-py2.txt /code/ -RUN pip install -r requirements-dev-py2.txt +ADD requirements-dev.txt /code/ +RUN pip install -r requirements-dev.txt RUN pip install tox==2.1.1 diff --git a/compose/container.py b/compose/container.py index f727c8673a..6f426532ac 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,9 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals -from functools import reduce - -import six +from six import iteritems +from six.moves import reduce from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_SERVICE @@ -90,7 +89,7 @@ class Container(object): private=private, **public[0]) return ', '.join(format_port(*item) - for item in sorted(six.iteritems(self.ports))) + for item in sorted(iteritems(self.ports))) @property def labels(self): diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 582c09fb9e..e2300fd4af 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -17,6 +17,8 @@ def stream_output(output, stream): diff = 0 for chunk in output: + if six.PY3 and not isinstance(chunk, str): + chunk = chunk.decode('utf-8') event = json.loads(chunk) all_events.append(event) diff --git a/compose/project.py b/compose/project.py index d14941e72b..cd88b2988b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -from functools import reduce from docker.errors import APIError +from six.moves import reduce from .config import ConfigurationError from .config import get_service_name_from_net diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..cc98422530 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +flake8 +tox diff --git a/script/test-versions b/script/test-versions index d67a6f5e12..e2102e449c 100755 --- a/script/test-versions +++ b/script/test-versions @@ -24,5 +24,5 @@ for version in $DOCKER_VERSIONS; do -e "DOCKER_DAEMON_ARGS" \ --entrypoint="script/dind" \ "$TAG" \ - script/wrapdocker nosetests --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html "$@" + script/wrapdocker tox "$@" done diff --git a/setup.py b/setup.py index b7fd440348..cdb5686cf3 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ tests_require = [ ] -if sys.version_info < (2, 6): +if sys.version_info < (2, 7): tests_require.append('unittest2') if sys.version_info[:1] < (3,): tests_require.append('pyinstaller') diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 609370a3e1..9552bf6a66 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -275,7 +275,7 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' user = 'sshd' - args = ['run', '--user={}'.format(user), name] + args = ['run', '--user={user}'.format(user=user), name] self.command.dispatch(args, None) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index fe54d4ae24..effd356dfa 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -358,7 +358,7 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(old_container.get('Volumes').keys(), ['/data']) + self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) volume_path = old_container.get('Volumes')['/data'] new_container, = service.execute_convergence_plan( diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 5674f4e4e8..e38a744353 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -8,7 +8,6 @@ from tests import unittest class ProgressStreamTestCase(unittest.TestCase): - def test_stream_output(self): output = [ '{"status": "Downloading", "progressDetail": {"current": ' diff --git a/tox.ini b/tox.ini index 35523a9697..2e3edd2a50 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ passenv = deps = -rrequirements.txt commands = - nosetests -v {posargs} + nosetests -v --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html {posargs} flake8 compose tests setup.py [testenv:pre-commit] @@ -38,6 +38,7 @@ deps = [testenv:py34] deps = {[testenv:py33]deps} +# TODO pypy3 [flake8] # ignore line-length for now From 2943ac6812bcc8cdcd5b877155cdf69dd08c5b8a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 2 Jul 2015 22:35:20 -0400 Subject: [PATCH 076/337] Cleanup requirements.txt so we don't have to maintain separate copies for py2 and py3. Signed-off-by: Daniel Nephin --- Dockerfile | 14 ++++++++++++++ MANIFEST.in | 2 +- compose/container.py | 7 ++++--- compose/project.py | 4 ++-- compose/service.py | 8 +++++--- compose/utils.py | 2 +- requirements-dev-py2.txt | 7 ------- requirements-dev-py3.txt | 2 -- requirements-dev.txt | 7 +++++-- script/test-versions | 2 +- setup.py | 3 --- tests/integration/resilience_test.py | 3 +-- tests/integration/service_test.py | 4 ++-- tests/unit/service_test.py | 2 +- tox.ini | 22 +++++++--------------- 15 files changed, 44 insertions(+), 45 deletions(-) delete mode 100644 requirements-dev-py2.txt delete mode 100644 requirements-dev-py3.txt diff --git a/Dockerfile b/Dockerfile index a4cc99fea0..546e28d69d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,18 @@ RUN set -ex; \ rm -rf /Python-2.7.9; \ rm Python-2.7.9.tgz +# Build python 3.4 from source +RUN set -ex; \ + curl -LO https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz; \ + tar -xzf Python-3.4.3.tgz; \ + cd Python-3.4.3; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-3.4.3; \ + rm Python-3.4.3.tgz + # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib @@ -63,6 +75,8 @@ RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ +RUN pip install tox + ADD requirements.txt /code/ RUN pip install -r requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 7420485961..7d48d347a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include Dockerfile include LICENSE include requirements.txt -include requirements-dev*.txt +include requirements-dev.txt include tox.ini include *.md include compose/config/schema.json diff --git a/compose/container.py b/compose/container.py index 6f426532ac..f727c8673a 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,8 +1,9 @@ from __future__ import absolute_import from __future__ import unicode_literals -from six import iteritems -from six.moves import reduce +from functools import reduce + +import six from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_SERVICE @@ -89,7 +90,7 @@ class Container(object): private=private, **public[0]) return ', '.join(format_port(*item) - for item in sorted(iteritems(self.ports))) + for item in sorted(six.iteritems(self.ports))) @property def labels(self): diff --git a/compose/project.py b/compose/project.py index cd88b2988b..a3127c6c29 100644 --- a/compose/project.py +++ b/compose/project.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +from functools import reduce from docker.errors import APIError -from six.moves import reduce from .config import ConfigurationError from .config import get_service_name_from_net @@ -340,7 +340,7 @@ class Project(object): self.service_names, ) - return filter(matches_service_names, containers) + return [c for c in containers if matches_service_names(c)] def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() diff --git a/compose/service.py b/compose/service.py index 647516ba84..d8a26e73a9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -709,7 +709,9 @@ class Service(object): def build(self, no_cache=False): log.info('Building %s...' % self.name) - path = six.binary_type(self.options['build']) + path = self.options['build'] + if not six.PY3: + path = path.encode('utf8') build_output = self.client.build( path=path, @@ -840,7 +842,7 @@ def merge_volume_bindings(volumes_option, previous_container): volume_bindings.update( get_container_data_volumes(previous_container, volumes_option)) - return volume_bindings.values() + return list(volume_bindings.values()) def get_container_data_volumes(container, volumes_option): @@ -853,7 +855,7 @@ def get_container_data_volumes(container, volumes_option): container_volumes = container.get('Volumes') or {} image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {} - for volume in set(volumes_option + image_volumes.keys()): + for volume in set(volumes_option + list(image_volumes)): volume = parse_volume_spec(volume) # No need to preserve host volumes if volume.external: diff --git a/compose/utils.py b/compose/utils.py index bd8922670e..0cbefba9be 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -97,5 +97,5 @@ def write_out_msg(stream, lines, msg_index, msg, status="done"): def json_hash(obj): dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) h = hashlib.sha256() - h.update(dump) + h.update(dump.encode('utf8')) return h.hexdigest() diff --git a/requirements-dev-py2.txt b/requirements-dev-py2.txt deleted file mode 100644 index 97fc4fed86..0000000000 --- a/requirements-dev-py2.txt +++ /dev/null @@ -1,7 +0,0 @@ -coverage==3.7.1 -flake8==2.3.0 -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller -mock >= 1.0.1 -nose==1.3.4 -pep8==1.6.1 -unittest2==0.8.0 diff --git a/requirements-dev-py3.txt b/requirements-dev-py3.txt deleted file mode 100644 index a2ba1c8b45..0000000000 --- a/requirements-dev-py3.txt +++ /dev/null @@ -1,2 +0,0 @@ -flake8 -nose >= 1.3.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index cc98422530..9e830733c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,5 @@ -flake8 -tox +flake8==2.3.0 +git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller +mock >= 1.0.1 +nose==1.3.4 +pep8==1.6.1 diff --git a/script/test-versions b/script/test-versions index e2102e449c..f39c17e819 100755 --- a/script/test-versions +++ b/script/test-versions @@ -24,5 +24,5 @@ for version in $DOCKER_VERSIONS; do -e "DOCKER_DAEMON_ARGS" \ --entrypoint="script/dind" \ "$TAG" \ - script/wrapdocker tox "$@" + script/wrapdocker tox -e py27,py34 -- "$@" done diff --git a/setup.py b/setup.py index cdb5686cf3..33335047bc 100644 --- a/setup.py +++ b/setup.py @@ -41,9 +41,7 @@ install_requires = [ tests_require = [ - 'mock >= 1.0.1', 'nose', - 'pyinstaller', 'flake8', ] @@ -51,7 +49,6 @@ tests_require = [ if sys.version_info < (2, 7): tests_require.append('unittest2') if sys.version_info[:1] < (3,): - tests_require.append('pyinstaller') tests_require.append('mock >= 1.0.1') diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index b1faf99dff..82a4680d8b 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -1,8 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -import mock - +from .. import mock from .testcases import DockerClientTestCase from compose.project import Project diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index effd356dfa..f300c6d531 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -363,7 +363,7 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - self.assertEqual(new_container.get('Volumes').keys(), ['/data']) + self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) def test_start_container_passes_through_options(self): @@ -498,7 +498,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - with open(os.path.join(base_dir, b'foo\xE2bar'), 'w') as f: + with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f: f.write("hello world\n") self.create_service('web', build=text_type(base_dir)).build() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 3bb3e1722b..f224752770 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -41,7 +41,7 @@ class ServiceTest(unittest.TestCase): dict(Name=str(i), Image='foo', Id=i) for i in range(3) ] service = Service('db', self.mock_client, 'myproject', image='foo') - self.assertEqual([c.id for c in service.containers()], range(3)) + self.assertEqual([c.id for c in service.containers()], list(range(3))) expected_labels = [ '{0}=myproject'.format(LABEL_PROJECT), diff --git a/tox.ini b/tox.ini index 2e3edd2a50..a2bd6b6b9b 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ envlist = py27,py34,pre-commit usedevelop=True passenv = LD_LIBRARY_PATH +setenv = + HOME=/tmp deps = -rrequirements.txt commands = @@ -19,26 +21,16 @@ commands = pre-commit install pre-commit run --all-files -[testenv:py26] -deps = - {[testenv]deps} - -rrequirements-dev-py2.txt - [testenv:py27] -deps = {[testenv:py26]deps} - -[testenv:pypy] -deps = {[testenv:py26]deps} - -[testenv:py33] deps = {[testenv]deps} - -rrequirements-dev-py3.txt + -rrequirements-dev.txt [testenv:py34] -deps = {[testenv:py33]deps} - -# TODO pypy3 +deps = + {[testenv]deps} + flake8 + nose [flake8] # ignore line-length for now From feaa4a5f1aa97caf984d08e50d4e6c384fe1f0ae Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 11:51:38 -0400 Subject: [PATCH 077/337] Unit tests passing again. Signed-off-by: Daniel Nephin --- compose/config/validation.py | 2 +- compose/service.py | 4 ++-- compose/utils.py | 4 ++-- tests/unit/config_test.py | 27 +++++++++++++-------------- tests/unit/service_test.py | 2 +- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 8911f5ae1d..e5f195f400 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -150,7 +150,7 @@ def process_errors(errors): config_key = error.path[0] required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) elif error.validator == 'dependencies': - dependency_key = error.validator_value.keys()[0] + dependency_key = list(error.validator_value.keys())[0] required_keys = ",".join(error.validator_value[dependency_key]) required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( dependency_key, service_name, dependency_key, required_keys)) diff --git a/compose/service.py b/compose/service.py index d8a26e73a9..9c0bc44391 100644 --- a/compose/service.py +++ b/compose/service.py @@ -103,11 +103,11 @@ class Service(object): def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) - containers = filter(None, [ + containers = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters=filters)]) + filters=filters)])) if not containers: check_for_legacy_containers( diff --git a/compose/utils.py b/compose/utils.py index 0cbefba9be..738fcacaff 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -3,11 +3,11 @@ import hashlib import json import logging import sys -from Queue import Empty -from Queue import Queue from threading import Thread from docker.errors import APIError +from six.moves.queue import Empty +from six.moves.queue import Queue log = logging.getLogger(__name__) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 7ecb6c4a2d..ccd5b57bf7 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -18,6 +18,10 @@ def make_service_dict(name, service_dict, working_dir): return config.ServiceLoader(working_dir=working_dir).make_service_dict(name, service_dict) +def service_sort(services): + return sorted(services, key=itemgetter('name')) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( @@ -32,8 +36,8 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual( - sorted(service_dicts, key=itemgetter('name')), - sorted([ + service_sort(service_dicts), + service_sort([ { 'name': 'bar', 'image': 'busybox', @@ -43,7 +47,7 @@ class ConfigTest(unittest.TestCase): 'name': 'foo', 'image': 'busybox', } - ], key=itemgetter('name')) + ]) ) def test_load_throws_error_when_not_dict(self): @@ -684,12 +688,7 @@ class ExtendsTest(unittest.TestCase): def test_extends(self): service_dicts = load_from_filename('tests/fixtures/extends/docker-compose.yml') - service_dicts = sorted( - service_dicts, - key=lambda sd: sd['name'], - ) - - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'name': 'mydb', 'image': 'busybox', @@ -706,7 +705,7 @@ class ExtendsTest(unittest.TestCase): "BAZ": "2", }, } - ]) + ])) def test_nested(self): service_dicts = load_from_filename('tests/fixtures/extends/nested.yml') @@ -728,7 +727,7 @@ class ExtendsTest(unittest.TestCase): We specify a 'file' key that is the filename we're already in. """ service_dicts = load_from_filename('tests/fixtures/extends/specify-file-as-self.yml') - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'environment': { @@ -749,7 +748,7 @@ class ExtendsTest(unittest.TestCase): 'image': 'busybox', 'name': 'web' } - ]) + ])) def test_circular(self): try: @@ -856,7 +855,7 @@ class ExtendsTest(unittest.TestCase): config is valid and correctly extends from itself. """ service_dicts = load_from_filename('tests/fixtures/extends/no-file-specified.yml') - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'name': 'myweb', 'image': 'busybox', @@ -872,7 +871,7 @@ class ExtendsTest(unittest.TestCase): "BAZ": "3", } } - ]) + ])) def test_blacklisted_options(self): def load_config(): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index f224752770..4708616e34 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -34,7 +34,7 @@ class ServiceTest(unittest.TestCase): def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') self.mock_client.containers.return_value = [] - self.assertEqual(service.containers(), []) + self.assertEqual(list(service.containers()), []) def test_containers_with_containers(self): self.mock_client.containers.return_value = [ From 7e4c3142d721ccab37ee6e34d93e9214fc3b89ef Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 12:43:08 -0400 Subject: [PATCH 078/337] Have log_printer use utf8 stream. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 6 +++--- tests/unit/log_printer_test.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index c7d0b638f8..034551ec62 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -5,7 +5,9 @@ import sys from itertools import cycle import six +from six import next +from compose import utils from . import colors from .multiplexer import Multiplexer from .utils import split_buffer @@ -17,13 +19,11 @@ class LogPrinter(object): self.attach_params = attach_params or {} self.prefix_width = self._calculate_prefix_width(containers) self.generators = self._make_log_generators(monochrome) - self.output = output + self.output = utils.get_output_stream(output) def run(self): mux = Multiplexer(self.generators) for line in mux.loop(): - if isinstance(line, six.text_type) and not six.PY3: - line = line.encode('utf-8') self.output.write(line) def _calculate_prefix_width(self, containers): diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py index f3fa64c614..284934a6af 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/log_printer_test.py @@ -32,7 +32,6 @@ class LogPrinterTest(unittest.TestCase): output = self.get_default_output() self.assertIn('\033[', output) - @unittest.skipIf(six.PY3, "Only test unicode in python2") def test_unicode(self): glyph = u'\u2022' @@ -42,7 +41,10 @@ class LogPrinterTest(unittest.TestCase): container = MockContainer(reader) output = run_log_printer([container]) - self.assertIn(glyph, output.decode('utf-8')) + if six.PY2: + output = output.decode('utf-8') + + self.assertIn(glyph, output) def run_log_printer(containers, monochrome=False): From 71ff872e8e6f09a15f39f90b7faba2b44201c46d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 13:16:13 -0400 Subject: [PATCH 079/337] Update unit tests for stream_output to match the behaviour of a docker-py response. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 3 +-- compose/progress_stream.py | 8 ++++---- compose/project.py | 4 ++-- compose/service.py | 2 ++ compose/utils.py | 9 ++++++++- tests/integration/legacy_test.py | 4 ++-- tests/unit/progress_stream_test.py | 18 +++++++++--------- tests/unit/service_test.py | 2 +- 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 034551ec62..69ada850e5 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,13 +4,12 @@ from __future__ import unicode_literals import sys from itertools import cycle -import six from six import next -from compose import utils from . import colors from .multiplexer import Multiplexer from .utils import split_buffer +from compose import utils class LogPrinter(object): diff --git a/compose/progress_stream.py b/compose/progress_stream.py index e2300fd4af..c44b33e561 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,8 +1,9 @@ -import codecs import json import six +from compose import utils + class StreamOutputError(Exception): pass @@ -10,14 +11,13 @@ class StreamOutputError(Exception): def stream_output(output, stream): is_terminal = hasattr(stream, 'isatty') and stream.isatty() - if not six.PY3: - stream = codecs.getwriter('utf-8')(stream) + stream = utils.get_output_stream(stream) all_events = [] lines = {} diff = 0 for chunk in output: - if six.PY3 and not isinstance(chunk, str): + if six.PY3: chunk = chunk.decode('utf-8') event = json.loads(chunk) all_events.append(event) diff --git a/compose/project.py b/compose/project.py index a3127c6c29..542c8785e5 100644 --- a/compose/project.py +++ b/compose/project.py @@ -324,11 +324,11 @@ class Project(object): else: service_names = self.service_names - containers = filter(None, [ + containers = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters={'label': self.labels(one_off=one_off)})]) + filters={'label': self.labels(one_off=one_off)})])) def matches_service_names(container): return container.labels.get(LABEL_SERVICE) in service_names diff --git a/compose/service.py b/compose/service.py index 9c0bc44391..a15ee1b9af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -710,6 +710,8 @@ class Service(object): log.info('Building %s...' % self.name) path = self.options['build'] + # python2 os.path() doesn't support unicode, so we need to encode it to + # a byte string if not six.PY3: path = path.encode('utf8') diff --git a/compose/utils.py b/compose/utils.py index 738fcacaff..c729228409 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -5,6 +5,7 @@ import logging import sys from threading import Thread +import six from docker.errors import APIError from six.moves.queue import Empty from six.moves.queue import Queue @@ -18,7 +19,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): For a given list of objects, call the callable passing in the first object we give it. """ - stream = codecs.getwriter('utf-8')(sys.stdout) + stream = get_output_stream() lines = [] errors = {} @@ -70,6 +71,12 @@ def parallel_execute(objects, obj_callable, msg_index, msg): stream.write("ERROR: for {} {} \n".format(error, errors[error])) +def get_output_stream(stream=sys.stdout): + if six.PY3: + return stream + return codecs.getwriter('utf-8')(stream) + + def write_out_msg(stream, lines, msg_index, msg, status="done"): """ Using special ANSI code characters we can write out the msg over the top of diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index fa983e6d59..3465d57f49 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,8 +1,8 @@ import unittest from docker.errors import APIError -from mock import Mock +from .. import mock from .testcases import DockerClientTestCase from compose import legacy from compose.project import Project @@ -66,7 +66,7 @@ class UtilitiesTestCase(unittest.TestCase): ) def test_get_legacy_containers(self): - client = Mock() + client = mock.Mock() client.containers.return_value = [ { "Id": "abc123", diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index e38a744353..d8f7ec8363 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -10,27 +10,27 @@ from tests import unittest class ProgressStreamTestCase(unittest.TestCase): def test_stream_output(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '31019763, "start": 1413653874, "total": 62763875}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'31019763, "start": 1413653874, "total": 62763875}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) def test_stream_output_div_zero(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '0, "start": 1413653874, "total": 0}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'0, "start": 1413653874, "total": 0}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) def test_stream_output_null_total(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '0, "start": 1413653874, "total": null}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'0, "start": 1413653874, "total": null}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4708616e34..275bde1bdd 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -280,7 +280,7 @@ class ServiceTest(unittest.TestCase): def test_build_does_not_pull(self): self.mock_client.build.return_value = [ - '{"stream": "Successfully built 12345"}', + b'{"stream": "Successfully built 12345"}', ] service = Service('foo', client=self.mock_client, build='.') From bd7c032a00b7701e5b29a983bb5a83b202dcd952 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 14:49:51 -0400 Subject: [PATCH 080/337] Fix service integration tests. Signed-off-by: Daniel Nephin --- compose/utils.py | 4 ++-- requirements-dev.txt | 1 + tests/integration/service_test.py | 16 +++++----------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index c729228409..30284f97bd 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -19,7 +19,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): For a given list of objects, call the callable passing in the first object we give it. """ - stream = get_output_stream() + stream = get_output_stream(sys.stdout) lines = [] errors = {} @@ -71,7 +71,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): stream.write("ERROR: for {} {} \n".format(error, errors[error])) -def get_output_stream(stream=sys.stdout): +def get_output_stream(stream): if six.PY3: return stream return codecs.getwriter('utf-8')(stream) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9e830733c1..c8a694ab27 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c mock >= 1.0.1 nose==1.3.4 pep8==1.6.1 +coverage==3.7.1 diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f300c6d531..bc9dcc6925 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -581,8 +581,7 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_scale_with_stopped_containers(self, mock_stdout): + def test_scale_with_stopped_containers(self): """ Given there are some stopped containers and scale is called with a desired number that is the same as the number of stopped containers, @@ -591,15 +590,11 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('web') next_number = service._next_container_number() valid_numbers = [next_number, next_number + 1] - service.create_container(number=next_number, quiet=True) - service.create_container(number=next_number + 1, quiet=True) + service.create_container(number=next_number) + service.create_container(number=next_number + 1) - for container in service.containers(): - self.assertFalse(container.is_running) - - service.scale(2) - - self.assertEqual(len(service.containers()), 2) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + service.scale(2) for container in service.containers(): self.assertTrue(container.is_running) self.assertTrue(container.number in valid_numbers) @@ -701,7 +696,6 @@ class ServiceTest(DockerClientTestCase): results in warning output. """ service = self.create_service('web', container_name='custom-container') - self.assertEqual(service.custom_container_name(), 'custom-container') service.scale(3) From 1451a6e1889b48a780759c41779e99b09ad16d18 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 18:54:59 -0400 Subject: [PATCH 081/337] Python3 requires a locale Signed-off-by: Daniel Nephin --- Dockerfile | 5 +++++ requirements-dev.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 546e28d69d..a9892031b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM debian:wheezy RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ + locales \ gcc \ make \ zlib1g \ @@ -61,6 +62,10 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz +# Python3 requires a valid locale +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 + ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.1 RUN set -ex; \ diff --git a/requirements-dev.txt b/requirements-dev.txt index c8a694ab27..adb4387df2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ +coverage==3.7.1 flake8==2.3.0 git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller mock >= 1.0.1 nose==1.3.4 pep8==1.6.1 -coverage==3.7.1 From 196f0afe8d689bf94543db97c96c17ad128459d6 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Tue, 25 Aug 2015 16:52:10 +0200 Subject: [PATCH 082/337] Update zsh completion with last changes - sdurrheimer/docker-compose-zsh-completion@4ff5c0d Add pause and unpause commands - sdurrheimer/docker-compose-zsh-completion@9948d66 Add -t/--timeout flag to scale command - sdurrheimer/docker-compose-zsh-completion@7cf14c8 Improve -p/--publish flag for the run command - sdurrheimer/docker-compose-zsh-completion@cb16818 Don't trigger expensive completion function for flags - sdurrheimer/docker-compose-zsh-completion@52d33fa Several cosmetic improvements and return responses - sdurrheimer/docker-compose-zsh-completion@632ca9c Bump to version 1.5.0 - sdurrheimer/docker-compose-zsh-completion@22f92d9 Refactor compose file and project-name option flags when invoking docker-compose - sdurrheimer/docker-compose-zsh-completion@1b512fc Refactor --help flags Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 154 +++++++++++++++---------- 1 file changed, 91 insertions(+), 63 deletions(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 9ac7e7560f..58105dc221 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -7,7 +7,7 @@ # ------------------------------------------------------------------------- # Version # ------- -# 0.1.0 +# 1.5.0 # ------------------------------------------------------------------------- # Authors # ------- @@ -37,40 +37,54 @@ __docker-compose_compose_file() { ___docker-compose_all_services_in_compose_file() { local already_selected local -a services - already_selected=$(echo ${words[@]} | tr " " "|") + already_selected=$(echo $words | tr " " "|") awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | grep -Ev "$already_selected" } # All services, even those without an existing container __docker-compose_services_all() { + [[ $PREFIX = -* ]] && return 1 + integer ret=1 services=$(___docker-compose_all_services_in_compose_file) - _alternative "args:services:($services)" + _alternative "args:services:($services)" && ret=0 + + return ret } # All services that have an entry with the given key in their docker-compose.yml section ___docker-compose_services_with_key() { local already_selected local -a buildable - already_selected=$(echo ${words[@]} | tr " " "|") + already_selected=$(echo $words | tr " " "|") # flatten sections to one line, then filter lines containing the key and return section name. awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' 2>/dev/null | grep -Ev "$already_selected" } # All services that are defined by a Dockerfile reference __docker-compose_services_from_build() { + [[ $PREFIX = -* ]] && return 1 + integer ret=1 buildable=$(___docker-compose_services_with_key build) - _alternative "args:buildable services:($buildable)" + _alternative "args:buildable services:($buildable)" && ret=0 + + return ret } # All services that are defined by an image __docker-compose_services_from_image() { + [[ $PREFIX = -* ]] && return 1 + integer ret=1 pullable=$(___docker-compose_services_with_key image) - _alternative "args:pullable services:($pullable)" + _alternative "args:pullable services:($pullable)" && ret=0 + + return ret } __docker-compose_get_services() { - local kind expl - declare -a running stopped lines args services + [[ $PREFIX = -* ]] && return 1 + integer ret=1 + local kind + declare -a running paused stopped lines args services docker_status=$(docker ps > /dev/null 2>&1) if [ $? -ne 0 ]; then @@ -80,64 +94,78 @@ __docker-compose_get_services() { kind=$1 shift - [[ $kind = (stopped|all) ]] && args=($args -a) + [[ $kind =~ (stopped|all) ]] && args=($args -a) - lines=(${(f)"$(_call_program commands docker ps ${args})"}) - services=(${(f)"$(_call_program commands docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)"}) + lines=(${(f)"$(_call_program commands docker ps $args)"}) + services=(${(f)"$(_call_program commands docker-compose 2>/dev/null $compose_options ps -q)"}) # Parse header line to find columns local i=1 j=1 k header=${lines[1]} declare -A begin end - while (( $j < ${#header} - 1 )) { - i=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 1)) - j=$(( $i + ${${header[$i,-1]}[(i) ]} - 1)) - k=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 2)) - begin[${header[$i,$(($j-1))]}]=$i - end[${header[$i,$(($j-1))]}]=$k - } + while (( j < ${#header} - 1 )); do + i=$(( j + ${${header[$j,-1]}[(i)[^ ]]} - 1 )) + j=$(( i + ${${header[$i,-1]}[(i) ]} - 1 )) + k=$(( j + ${${header[$j,-1]}[(i)[^ ]]} - 2 )) + begin[${header[$i,$((j-1))]}]=$i + end[${header[$i,$((j-1))]}]=$k + done lines=(${lines[2,-1]}) # Container ID local line s name local -a names for line in $lines; do - if [[ $services == *"${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}"* ]]; then + if [[ ${services[@]} == *"${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}"* ]]; then names=(${(ps:,:)${${line[${begin[NAMES]},-1]}%% *}}) for name in $names; do s="${${name%_*}#*_}:${(l:15:: :::)${${line[${begin[CREATED]},${end[CREATED]}]/ ago/}%% ##}}" s="$s, ${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}" - s="$s, ${${${line[$begin[IMAGE],$end[IMAGE]]}/:/\\:}%% ##}" + s="$s, ${${${line[${begin[IMAGE]},${end[IMAGE]}]}/:/\\:}%% ##}" if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = Exit* ]]; then stopped=($stopped $s) else + if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = *\(Paused\)* ]]; then + paused=($paused $s) + fi running=($running $s) fi done fi done - [[ $kind = (running|all) ]] && _describe -t services-running "running services" running - [[ $kind = (stopped|all) ]] && _describe -t services-stopped "stopped services" stopped + [[ $kind =~ (running|all) ]] && _describe -t services-running "running services" running "$@" && ret=0 + [[ $kind =~ (paused|all) ]] && _describe -t services-paused "paused services" paused "$@" && ret=0 + [[ $kind =~ (stopped|all) ]] && _describe -t services-stopped "stopped services" stopped "$@" && ret=0 + + return ret +} + +__docker-compose_pausedservices() { + [[ $PREFIX = -* ]] && return 1 + __docker-compose_get_services paused "$@" } __docker-compose_stoppedservices() { + [[ $PREFIX = -* ]] && return 1 __docker-compose_get_services stopped "$@" } __docker-compose_runningservices() { + [[ $PREFIX = -* ]] && return 1 __docker-compose_get_services running "$@" } -__docker-compose_services () { +__docker-compose_services() { + [[ $PREFIX = -* ]] && return 1 __docker-compose_get_services all "$@" } __docker-compose_caching_policy() { - oldp=( "$1"(Nmh+1) ) # 1 hour + oldp=( "$1"(Nmh+1) ) # 1 hour (( $#oldp )) } -__docker-compose_commands () { +__docker-compose_commands() { local cache_policy zstyle -s ":completion:${curcontext}:" cache-policy cache_policy @@ -156,13 +184,14 @@ __docker-compose_commands () { _describe -t docker-compose-commands "docker-compose command" _docker_compose_subcommands } -__docker-compose_subcommand () { - local -a _command_args +__docker-compose_subcommand() { + local opts_help='(: -)--help[Print usage]' integer ret=1 + case "$words[1]" in (build) _arguments \ - '--help[Print usage]' \ + $opts_help \ '--no-cache[Do not use cache when building the image]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; @@ -171,24 +200,29 @@ __docker-compose_subcommand () { ;; (kill) _arguments \ - '--help[Print usage]' \ + $opts_help \ '-s[SIGNAL to send to the container. Default signal is SIGKILL.]:signal:_signals' \ '*:running services:__docker-compose_runningservices' && ret=0 ;; (logs) _arguments \ - '--help[Print usage]' \ + $opts_help \ '--no-color[Produce monochrome output.]' \ '*:services:__docker-compose_services_all' && ret=0 ;; (migrate-to-labels) _arguments -A '-*' \ - '--help[Print usage]' \ + $opts_help \ '(-):Recreate containers to add labels' && ret=0 ;; + (pause) + _arguments \ + $opts_help \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; (port) _arguments \ - '--help[Print usage]' \ + $opts_help \ '--protocol=-[tcp or udap (defaults to tcp)]:protocol:(tcp udp)' \ '--index=-[index of the container if there are mutiple instances of a service (defaults to 1)]:index: ' \ '1:running services:__docker-compose_runningservices' \ @@ -196,33 +230,33 @@ __docker-compose_subcommand () { ;; (ps) _arguments \ - '--help[Print usage]' \ + $opts_help \ '-q[Only display IDs]' \ '*:services:__docker-compose_services_all' && ret=0 ;; (pull) _arguments \ - '--help[Print usage]' \ + $opts_help \ '*:services:__docker-compose_services_from_image' && ret=0 ;; (rm) _arguments \ + $opts_help \ '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ - '--help[Print usage]' \ '-v[Remove volumes associated with containers]' \ '*:stopped services:__docker-compose_stoppedservices' && ret=0 ;; (run) _arguments \ + $opts_help \ '-d[Detached mode: Run container in the background, print new container name.]' \ '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ - '--help[Print usage]' \ '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ "--no-deps[Don't start linked services.]" \ '--rm[Remove container after run. Ignored in detached mode.]' \ - "--publish[Run command with manually mapped container's port(s) to the host.]" \ "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ + '(-p --publish)'{-p,--publish=-}"[Run command with manually mapped container's port(s) to the host.]" \ '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ '(-):services:__docker-compose_services' \ '(-):command: _command_names -e' \ @@ -230,45 +264,52 @@ __docker-compose_subcommand () { ;; (scale) _arguments \ - '--help[Print usage]' \ + $opts_help \ + '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ '*:running services:__docker-compose_runningservices' && ret=0 ;; (start) _arguments \ - '--help[Print usage]' \ + $opts_help \ '*:stopped services:__docker-compose_stoppedservices' && ret=0 ;; (stop|restart) _arguments \ - '--help[Print usage]' \ + $opts_help \ '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ '*:running services:__docker-compose_runningservices' && ret=0 ;; + (unpause) + _arguments \ + $opts_help \ + '*:paused services:__docker-compose_pausedservices' && ret=0 + ;; (up) _arguments \ + $opts_help \ '-d[Detached mode: Run containers in the background, print new container names.]' \ - '--help[Print usage]' \ '--no-color[Produce monochrome output.]' \ "--no-deps[Don't start linked services.]" \ + "--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" \ "--no-recreate[If containers already exist, don't recreate them.]" \ - "--force-recreate[Recreate containers even if their configuration and image haven't changed]" \ "--no-build[Don't build an image, even if it's missing]" \ '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ '*:services:__docker-compose_services_all' && ret=0 ;; (version) _arguments \ - '--help[Print usage]' \ + $opts_help \ "--short[Shows only Compose's version number.]" && ret=0 ;; (*) - _message 'Unknown sub command' + _message 'Unknown sub command' && ret=1 + ;; esac return ret } -_docker-compose () { +_docker-compose() { # Support for subservices, which allows for `compdef _docker docker-shell=_docker_containers`. # Based on /usr/share/zsh/functions/Completion/Unix/_git without support for `ret`. if [[ $service != docker-compose ]]; then @@ -276,7 +317,8 @@ _docker-compose () { return fi - local curcontext="$curcontext" state line ret=1 + local curcontext="$curcontext" state line + integer ret=1 typeset -A opt_args _arguments -C \ @@ -288,23 +330,9 @@ _docker-compose () { '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 - local counter=1 - #local compose_file compose_project - while [ $counter -lt ${#words[@]} ]; do - case "${words[$counter]}" in - -f|--file) - (( counter++ )) - compose_file="${words[$counter]}" - ;; - -p|--project-name) - (( counter++ )) - compose_project="${words[$counter]}" - ;; - *) - ;; - esac - (( counter++ )) - done + local compose_file=${opt_args[-f]}${opt_args[--file]} + local compose_project=${opt_args[-p]}${opt_args[--project-name]} + local compose_options="${compose_file:+--file $compose_file} ${compose_project:+--project-name $compose_project}" case $state in (command) From 2b589606dab4e9acc94c15fd328760e4da0a84cd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 25 Aug 2015 16:49:49 -0400 Subject: [PATCH 083/337] Move log_printer_test into correct testing module for naming convention. Signed-off-by: Daniel Nephin --- tests/unit/{ => cli}/log_printer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/unit/{ => cli}/log_printer_test.py (98%) diff --git a/tests/unit/log_printer_test.py b/tests/unit/cli/log_printer_test.py similarity index 98% rename from tests/unit/log_printer_test.py rename to tests/unit/cli/log_printer_test.py index 284934a6af..142bd7f342 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -5,8 +5,8 @@ import os import six -from .. import unittest from compose.cli.log_printer import LogPrinter +from tests import unittest class LogPrinterTest(unittest.TestCase): From a348993d2c15688aefb05914b5e3973622f83179 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 25 Aug 2015 16:55:29 -0400 Subject: [PATCH 084/337] Remove two unused functions from cli/utils.py Signed-off-by: Daniel Nephin --- compose/cli/utils.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index b6c83f9e1f..cbc9123cf2 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -2,7 +2,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -import datetime import os import platform import ssl @@ -36,39 +35,6 @@ def yesno(prompt, default=None): return None -# http://stackoverflow.com/a/5164027 -def prettydate(d): - diff = datetime.datetime.utcnow() - d - s = diff.seconds - if diff.days > 7 or diff.days < 0: - return d.strftime('%d %b %y') - elif diff.days == 1: - return '1 day ago' - elif diff.days > 1: - return '{0} days ago'.format(diff.days) - elif s <= 1: - return 'just now' - elif s < 60: - return '{0} seconds ago'.format(s) - elif s < 120: - return '1 minute ago' - elif s < 3600: - return '{0} minutes ago'.format(s / 60) - elif s < 7200: - return '1 hour ago' - else: - return '{0} hours ago'.format(s / 3600) - - -def mkdir(path, permissions=0o700): - if not os.path.exists(path): - os.mkdir(path) - - os.chmod(path, permissions) - - return path - - def find_candidates_in_parent_dirs(filenames, path): """ Given a directory path to start, looks for filenames in the From 9d9550c5b677647ad52235ed6d7fcf5ccfeac21a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 25 Aug 2015 17:17:12 -0400 Subject: [PATCH 085/337] Fix log printing for python3 by converting everything to unicode. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 2 +- compose/cli/utils.py | 7 ++++--- tests/unit/cli/log_printer_test.py | 5 ++--- tests/unit/split_buffer_test.py | 28 ++++++++++++++-------------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 69ada850e5..c2fcc54fdd 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -57,7 +57,7 @@ class LogPrinter(object): def _make_log_generator(self, container, color_fn): prefix = color_fn(self._generate_prefix(container)) # Attach to container before log printer starts running - line_generator = split_buffer(self._attach(container), '\n') + line_generator = split_buffer(self._attach(container), u'\n') for line in line_generator: yield prefix + line diff --git a/compose/cli/utils.py b/compose/cli/utils.py index cbc9123cf2..0b7ac683d1 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -7,6 +7,7 @@ import platform import ssl import subprocess +import six from docker import version as docker_py_version from six.moves import input @@ -63,11 +64,11 @@ def split_buffer(reader, separator): separator, except for the last one if none was found on the end of the input. """ - buffered = str('') - separator = str(separator) + buffered = six.text_type('') + separator = six.text_type(separator) for data in reader: - buffered += data + buffered += data.decode('utf-8') while True: index = buffered.find(separator) if index == -1: diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 142bd7f342..d8fbf94b9a 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -12,7 +12,7 @@ from tests import unittest class LogPrinterTest(unittest.TestCase): def get_default_output(self, monochrome=False): def reader(*args, **kwargs): - yield "hello\nworld" + yield b"hello\nworld" container = MockContainer(reader) output = run_log_printer([container], monochrome=monochrome) @@ -36,11 +36,10 @@ class LogPrinterTest(unittest.TestCase): glyph = u'\u2022' def reader(*args, **kwargs): - yield glyph + '\n' + yield glyph.encode('utf-8') + b'\n' container = MockContainer(reader) output = run_log_printer([container]) - if six.PY2: output = output.decode('utf-8') diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index 1164609937..47c72f0865 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -8,33 +8,33 @@ from compose.cli.utils import split_buffer class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): - yield 'abc\n' - yield 'def\n' - yield 'ghi\n' + yield b'abc\n' + yield b'def\n' + yield b'ghi\n' self.assert_produces(reader, ['abc\n', 'def\n', 'ghi\n']) def test_no_end_separator(self): def reader(): - yield 'abc\n' - yield 'def\n' - yield 'ghi' + yield b'abc\n' + yield b'def\n' + yield b'ghi' self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_multiple_line_chunk(self): def reader(): - yield 'abc\ndef\nghi' + yield b'abc\ndef\nghi' self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_chunked_line(self): def reader(): - yield 'a' - yield 'b' - yield 'c' - yield '\n' - yield 'd' + yield b'a' + yield b'b' + yield b'c' + yield b'\n' + yield b'd' self.assert_produces(reader, ['abc\n', 'd']) @@ -42,12 +42,12 @@ class SplitBufferTest(unittest.TestCase): string = u"a\u2022c\n" def reader(): - yield string + yield string.encode('utf-8') self.assert_produces(reader, [string]) def assert_produces(self, reader, expectations): - split = split_buffer(reader(), '\n') + split = split_buffer(reader(), u'\n') for (actual, expected) in zip(split, expectations): self.assertEqual(type(actual), type(expected)) From 34249fad5d2e14320e35aaa1bf3cc8b3843e69c7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 25 Aug 2015 14:26:33 +0100 Subject: [PATCH 086/337] Improve release docs Incorporating questions from https://gist.github.com/aanand/e567bd8d6a5d8e28c829 Signed-off-by: Aanand Prasad --- RELEASE_PROCESS.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index e81a55ec69..0d5f42eba0 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -18,6 +18,12 @@ Building a Compose release 4. Write release notes in `CHANGES.md`. + Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context where appropriate. + + Bug fixes are worth mentioning if it's likely that they've affected lots of people, or if they were regressions in the previous version. + + Improvements to the code are not worth mentioning. + 5. Add a bump commit: git commit -am "Bump $VERSION" @@ -78,9 +84,29 @@ Building a Compose release git push git@github.com:docker/compose.git $TAG -8. Create a release from the tag on GitHub. +8. Draft a release from the tag on GitHub. -9. Paste in installation instructions and release notes. + - Go to https://github.com/docker/compose/releases and click "Draft a new release". + - In the "Tag version" dropdown, select the tag you just pushed. + +9. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: + + Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. + + Secondly, if you're a Mac user, the **[Docker Toolbox](https://www.docker.com/toolbox)** will install Compose ${VERSION} for you, alongside the latest versions of the Docker Engine, Machine and Kitematic. + + Otherwise, you can use the usual commands to install/upgrade. Either download the binary: + + curl -L https://github.com/docker/compose/releases/download/1.5.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + + Or install the PyPi package: + + pip install -U docker-compose==1.5.0 + + Here's what's new: + + ...release notes go here... 10. Attach the binaries. From 7e22719090bf33012c5cd327cc70ebd965cd923f Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 26 Aug 2015 17:48:51 +0200 Subject: [PATCH 087/337] Fix suppressed blank in completion of `scale --help` Wrong placement of `compopt -o` introduces an unexpected behavior that did not matter as long as --help was the only option (you would probably not continue to type after --help): completion of options would not automatically append a whitespace character as expected. For the outstanding addition of the --timeout option, which has an argument, this would mean that the user would have to type an extra whitespace after completion of --timeout before the argument could be added. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 5692f0e4b8..ee810ffaef 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -278,10 +278,7 @@ _docker_compose_scale() { case "$prev" in =) COMPREPLY=("$cur") - ;; - *) - COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) - compopt -o nospace + return ;; esac @@ -289,6 +286,10 @@ _docker_compose_scale() { -*) COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) ;; + *) + COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) + compopt -o nospace + ;; esac } From b03a2f79104e098a9e9a01f7fcb9e8da7e6c4ec4 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 26 Aug 2015 17:57:04 +0200 Subject: [PATCH 088/337] Add completion for `scale --timeout` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index ee810ffaef..71745d8270 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -280,11 +280,14 @@ _docker_compose_scale() { COMPREPLY=("$cur") return ;; + --timeout|-t) + return + ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --timeout -t" -- "$cur" ) ) ;; *) COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) From 2cfda01ff4562304d646b10c9069683bda774e33 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 26 Aug 2015 18:16:36 +0200 Subject: [PATCH 089/337] Use consistent argument order in bash completion In most of this file and in Dockers's bash completion the sort order of options is to sort alphabetically by long option name. The short options are put right behind their long couterpart. This commit improves consistency. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 71745d8270..fe46a334ed 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -223,7 +223,7 @@ _docker_compose_pull() { _docker_compose_restart() { case "$prev" in - -t | --timeout) + --timeout|-t) return ;; esac @@ -311,7 +311,7 @@ _docker_compose_start() { _docker_compose_stop() { case "$prev" in - -t | --timeout) + --timeout|-t) return ;; esac @@ -341,7 +341,7 @@ _docker_compose_unpause() { _docker_compose_up() { case "$prev" in - -t | --timeout) + --timeout|-t) return ;; esac @@ -402,11 +402,11 @@ _docker_compose() { local compose_file compose_project while [ $counter -lt $cword ]; do case "${words[$counter]}" in - -f|--file) + --file|-f) (( counter++ )) compose_file="${words[$counter]}" ;; - -p|--project-name) + --project-name|p) (( counter++ )) compose_project="${words[$counter]}" ;; From 54973e8200a13dc9a386464c8eddaa2790155326 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 26 Aug 2015 12:53:11 -0400 Subject: [PATCH 090/337] Remove flake8 ignores and wrap the longest lines to 140 char. Signed-off-by: Daniel Nephin --- compose/config/validation.py | 34 +++++++++++++++++++++++++------ compose/legacy.py | 3 ++- compose/project.py | 16 ++++++++++++--- tests/integration/cli_test.py | 4 +++- tests/integration/service_test.py | 5 ++++- tests/unit/config_test.py | 5 ++++- tests/unit/container_test.py | 10 ++++++++- tox.ini | 4 ++-- 8 files changed, 65 insertions(+), 16 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index e5f195f400..0df73e3c25 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -30,7 +30,11 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' -@FormatChecker.cls_checks(format="ports", raises=ValidationError("Invalid port formatting, it should be '[[remote_ip:]remote_port:]port[/protocol]'")) +@FormatChecker.cls_checks( + format="ports", + raises=ValidationError( + "Invalid port formatting, it should be " + "'[[remote_ip:]remote_port:]port[/protocol]'")) def format_ports(instance): try: split_port(instance) @@ -122,9 +126,14 @@ def process_errors(errors): invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) elif error.validator == 'anyOf': if 'image' in error.instance and 'build' in error.instance: - required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name)) + required.append( + "Service '{}' has both an image and build path specified. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) elif 'image' not in error.instance and 'build' not in error.instance: - required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name)) + required.append( + "Service '{}' has neither an image nor a build path " + "specified. Exactly one must be provided.".format(service_name)) else: required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': @@ -143,12 +152,25 @@ def process_errors(errors): if len(error.path) > 0: config_key = " ".join(["'%s'" % k for k in error.path]) - type_errors.append("Service '{}' configuration key {} contains an invalid type, it should be {} {}".format(service_name, config_key, msg, error.validator_value)) + type_errors.append( + "Service '{}' configuration key {} contains an invalid " + "type, it should be {} {}".format( + service_name, + config_key, + msg, + error.validator_value)) else: - root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(service_name)) + root_msgs.append( + "Service '{}' doesn\'t have any configuration options. " + "All top level keys in your docker-compose.yml must map " + "to a dictionary of configuration options.'".format(service_name)) elif error.validator == 'required': config_key = error.path[0] - required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) + required.append( + "Service '{}' option '{}' is invalid, {}".format( + service_name, + config_key, + _clean_error_message(error.message))) elif error.validator == 'dependencies': dependency_key = list(error.validator_value.keys())[0] required_keys = ",".join(error.validator_value[dependency_key]) diff --git a/compose/legacy.py b/compose/legacy.py index e8f4f95739..5416241789 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -17,7 +17,8 @@ Compose found the following containers without labels: {names_list} -As of Compose 1.3.0, containers are identified with labels instead of naming convention. If you want to continue using these containers, run: +As of Compose 1.3.0, containers are identified with labels instead of naming +convention. If you want to continue using these containers, run: $ docker-compose migrate-to-labels diff --git a/compose/project.py b/compose/project.py index 542c8785e5..4e8696ba88 100644 --- a/compose/project.py +++ b/compose/project.py @@ -157,7 +157,9 @@ class Project(object): try: links.append((self.get_service(service_name), link_name)) except NoSuchService: - raise ConfigurationError('Service "%s" has a link to service "%s" which does not exist.' % (service_dict['name'], service_name)) + raise ConfigurationError( + 'Service "%s" has a link to service "%s" which does not ' + 'exist.' % (service_dict['name'], service_name)) del service_dict['links'] return links @@ -173,7 +175,11 @@ class Project(object): container = Container.from_id(self.client, volume_name) volumes_from.append(container) except APIError: - raise ConfigurationError('Service "%s" mounts volumes from "%s", which is not the name of a service or container.' % (service_dict['name'], volume_name)) + raise ConfigurationError( + 'Service "%s" mounts volumes from "%s", which is ' + 'not the name of a service or container.' % ( + service_dict['name'], + volume_name)) del service_dict['volumes_from'] return volumes_from @@ -188,7 +194,11 @@ class Project(object): try: net = Container.from_id(self.client, net_name) except APIError: - raise ConfigurationError('Service "%s" is trying to use the network of "%s", which is not the name of a service or container.' % (service_dict['name'], net_name)) + raise ConfigurationError( + 'Service "%s" is trying to use the network of "%s", ' + 'which is not the name of a service or container.' % ( + service_dict['name'], + net_name)) else: net = service_dict['net'] diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 9552bf6a66..a7bc3b49a2 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -93,7 +93,9 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_digest(self, mock_logging): self.command.dispatch(['-f', 'digest.yml', 'pull'], None) mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') - mock_logging.info.assert_any_call('Pulling digest (busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') + mock_logging.info.assert_any_call( + 'Pulling digest (busybox@' + 'sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') @mock.patch('sys.stdout', new_callable=StringIO) def test_build_no_cache(self, mock_stdout): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index bc9dcc6925..fc634c8ca3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -804,7 +804,10 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(env[k], v) def test_env_from_file_combined_with_env(self): - service = self.create_service('web', environment=['ONE=1', 'TWO=2', 'THREE=3'], env_file=['tests/fixtures/env/one.env', 'tests/fixtures/env/two.env']) + service = self.create_service( + 'web', + environment=['ONE=1', 'TWO=2', 'THREE=3'], + env_file=['tests/fixtures/env/one.env', 'tests/fixtures/env/two.env']) env = create_and_start_container(service).environment for k, v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): self.assertEqual(env[k], v) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index ccd5b57bf7..51dac052f3 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -529,7 +529,10 @@ class MemoryOptionsTest(unittest.TestCase): When you set a 'memswap_limit' it is invalid config unless you also set a mem_limit """ - expected_error_msg = "Invalid 'memswap_limit' configuration for 'foo' service: when defining 'memswap_limit' you must set 'mem_limit' as well" + expected_error_msg = ( + "Invalid 'memswap_limit' configuration for 'foo' service: when " + "defining 'memswap_limit' you must set 'mem_limit' as well" + ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 1eba9f656d..5637330cd4 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -142,4 +142,12 @@ class GetContainerNameTestCase(unittest.TestCase): self.assertIsNone(get_container_name({})) self.assertEqual(get_container_name({'Name': 'myproject_db_1'}), 'myproject_db_1') self.assertEqual(get_container_name({'Names': ['/myproject_db_1', '/myproject_web_1/db']}), 'myproject_db_1') - self.assertEqual(get_container_name({'Names': ['/swarm-host-1/myproject_db_1', '/swarm-host-1/myproject_web_1/db']}), 'myproject_db_1') + self.assertEqual( + get_container_name({ + 'Names': [ + '/swarm-host-1/myproject_db_1', + '/swarm-host-1/myproject_web_1/db' + ] + }), + 'myproject_db_1' + ) diff --git a/tox.ini b/tox.ini index a2bd6b6b9b..4b27a4e9be 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,6 @@ deps = nose [flake8] -# ignore line-length for now -ignore = E501,E203 +# Allow really long lines for now +max-line-length = 140 exclude = compose/packages From bdec7e6b52500ce7ec3d0f0ee8acc789182e1e10 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 26 Aug 2015 13:32:21 -0400 Subject: [PATCH 091/337] Cleanup some test case, remove unused mock return values, and use standard single underscore for unused variable Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 8 ++++---- tests/unit/service_test.py | 6 ------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 9552bf6a66..78ff7604c7 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -313,7 +313,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual('moto=bobo', container.environment['allo']) @mock.patch('dockerpty.start') - def test_run_service_without_map_ports(self, __): + def test_run_service_without_map_ports(self, _): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' self.command.dispatch(['run', '-d', 'simple'], None) @@ -331,7 +331,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_assigned, None) @mock.patch('dockerpty.start') - def test_run_service_with_map_ports(self, __): + def test_run_service_with_map_ports(self, _): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' @@ -354,7 +354,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[1], "0.0.0.0:49154") @mock.patch('dockerpty.start') - def test_run_service_with_explicitly_maped_ports(self, __): + def test_run_service_with_explicitly_maped_ports(self, _): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' @@ -373,7 +373,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_full, "0.0.0.0:30001") @mock.patch('dockerpty.start') - def test_run_service_with_explicitly_maped_ip_ports(self, __): + def test_run_service_with_explicitly_maped_ip_ports(self, _): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 275bde1bdd..5d37bfedf1 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -99,14 +99,12 @@ class ServiceTest(unittest.TestCase): def test_split_domainname_none(self): service = Service('foo', image='foo', hostname='name', client=self.mock_client) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertFalse('domainname' in opts, 'domainname') def test_memory_swap_limit(self): service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'some': 'overrides'}, 1) self.assertEqual(opts['host_config']['MemorySwap'], 2000000000) self.assertEqual(opts['host_config']['Memory'], 1000000000) @@ -114,7 +112,6 @@ class ServiceTest(unittest.TestCase): def test_log_opt(self): log_opt = {'syslog-address': 'tcp://192.168.0.42:123'} service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'some': 'overrides'}, 1) self.assertIsInstance(opts['host_config']['LogConfig'], LogConfig) @@ -127,7 +124,6 @@ class ServiceTest(unittest.TestCase): hostname='name.domain.tld', image='foo', client=self.mock_client) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') @@ -139,7 +135,6 @@ class ServiceTest(unittest.TestCase): image='foo', domainname='domain.tld', client=self.mock_client) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') @@ -151,7 +146,6 @@ class ServiceTest(unittest.TestCase): domainname='domain.tld', image='foo', client=self.mock_client) - self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name.sub', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') From d2718bed9938ad3d72500fb722c02970de4d1ac8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 26 Aug 2015 13:33:03 -0400 Subject: [PATCH 092/337] Allow setting a one-off container name Signed-off-by: Daniel Nephin --- compose/cli/main.py | 4 ++++ compose/service.py | 2 +- tests/integration/cli_test.py | 10 ++++++++++ tests/unit/cli_test.py | 4 ++++ tests/unit/service_test.py | 13 +++++++++++++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 890a3c3717..58e5428516 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -308,6 +308,7 @@ class TopLevelCommand(Command): --allow-insecure-ssl Deprecated - no effect. -d Detached mode: Run container in the background, print new container name. + --name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. -e KEY=VAL Set an environment variable (can be used multiple times) -u, --user="" Run as specified username or uid @@ -374,6 +375,9 @@ class TopLevelCommand(Command): 'can not be used togather' ) + if options['--name']: + container_options['name'] = options['--name'] + try: container = service.create_container( quiet=True, diff --git a/compose/service.py b/compose/service.py index a15ee1b9af..a0423ff447 100644 --- a/compose/service.py +++ b/compose/service.py @@ -586,7 +586,7 @@ class Service(object): if self.custom_container_name() and not one_off: container_options['name'] = self.custom_container_name() - else: + elif not container_options.get('name'): container_options['name'] = self.get_container_name(number, one_off) if add_config_hash: diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 78ff7604c7..b94d7d1eec 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -391,6 +391,16 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "127.0.0.1:30000") self.assertEqual(port_full, "127.0.0.1:30001") + @mock.patch('dockerpty.start') + def test_run_with_custom_name(self, _): + self.command.base_dir = 'tests/fixtures/environment-composefile' + name = 'the-container-name' + self.command.dispatch(['run', '--name', name, 'service'], None) + + service = self.project.get_service('service') + container, = service.containers(stopped=True, one_off=True) + self.assertEqual(container.name, name) + def test_rm(self): service = self.project.get_service('simple') service.create_container() diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 7d22ad02ff..1fd9f529ed 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -112,6 +112,7 @@ class CLITestCase(unittest.TestCase): '--service-ports': None, '--publish': [], '--rm': None, + '--name': None, }) _, _, call_kwargs = mock_client.create_container.mock_calls[0] @@ -141,6 +142,7 @@ class CLITestCase(unittest.TestCase): '--service-ports': None, '--publish': [], '--rm': None, + '--name': None, }) _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertEquals(call_kwargs['host_config']['RestartPolicy']['Name'], 'always') @@ -166,6 +168,7 @@ class CLITestCase(unittest.TestCase): '--service-ports': None, '--publish': [], '--rm': True, + '--name': None, }) _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertFalse('RestartPolicy' in call_kwargs['host_config']) @@ -195,4 +198,5 @@ class CLITestCase(unittest.TestCase): '--service-ports': True, '--publish': ['80:80'], '--rm': None, + '--name': None, }) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5d37bfedf1..a24e524dd5 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -150,6 +150,19 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['hostname'], 'name.sub', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') + def test_get_container_create_options_with_name_option(self): + service = Service( + 'foo', + image='foo', + client=self.mock_client, + container_name='foo1') + name = 'the_new_name' + opts = service._get_container_create_options( + {'name': name}, + 1, + one_off=True) + self.assertEqual(opts['name'], name) + def test_get_container_not_found(self): self.mock_client.containers.return_value = [] service = Service('foo', client=self.mock_client, image='foo') From 3a0153859ae6624970696c967e18a99710479cd4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 26 Aug 2015 16:21:31 -0400 Subject: [PATCH 093/337] Resolves #1856, fix regression in #1645. Includes some refactoring to make testing easier. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 36 ++++++++++++++++++---------- compose/container.py | 6 ++++- tests/unit/cli/main_test.py | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 tests/unit/cli/main_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 58e5428516..2ace13c22b 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -538,19 +538,8 @@ class TopLevelCommand(Command): ) if not detached: - print("Attaching to", list_containers(to_attach)) - log_printer = LogPrinter(to_attach, attach_params={"logs": True}, monochrome=monochrome) - - try: - log_printer.run() - finally: - def handler(signal, frame): - project.kill(service_names=service_names) - sys.exit(0) - signal.signal(signal.SIGINT, handler) - - print("Gracefully stopping... (press Ctrl+C again to force)") - project.stop(service_names=service_names, timeout=timeout) + log_printer = build_log_printer(to_attach, service_names, monochrome) + attach_to_logs(project, log_printer, service_names, timeout) def migrate_to_labels(self, project, _options): """ @@ -593,5 +582,26 @@ class TopLevelCommand(Command): print(get_version_info('full')) +def build_log_printer(containers, service_names, monochrome): + return LogPrinter( + [c for c in containers if c.service in service_names], + attach_params={"logs": True}, + monochrome=monochrome) + + +def attach_to_logs(project, log_printer, service_names, timeout): + print("Attaching to", list_containers(log_printer.containers)) + try: + log_printer.run() + finally: + def handler(signal, frame): + project.kill(service_names=service_names) + sys.exit(0) + signal.signal(signal.SIGINT, handler) + + print("Gracefully stopping... (press Ctrl+C again to force)") + project.stop(service_names=service_names, timeout=timeout) + + def list_containers(containers): return ", ".join(c.name for c in containers) diff --git a/compose/container.py b/compose/container.py index f727c8673a..f2d8a403e1 100644 --- a/compose/container.py +++ b/compose/container.py @@ -64,9 +64,13 @@ class Container(object): def name(self): return self.dictionary['Name'][1:] + @property + def service(self): + return self.labels.get(LABEL_SERVICE) + @property def name_without_project(self): - return '{0}_{1}'.format(self.labels.get(LABEL_SERVICE), self.number) + return '{0}_{1}'.format(self.service, self.number) @property def number(self): diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py new file mode 100644 index 0000000000..817e8f49b6 --- /dev/null +++ b/tests/unit/cli/main_test.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import + +from compose import container +from compose.cli.log_printer import LogPrinter +from compose.cli.main import attach_to_logs +from compose.cli.main import build_log_printer +from compose.project import Project +from tests import mock +from tests import unittest + + +def mock_container(service, number): + return mock.create_autospec( + container.Container, + service=service, + number=number, + name_without_project='{0}_{1}'.format(service, number)) + + +class CLIMainTestCase(unittest.TestCase): + + def test_build_log_printer(self): + containers = [ + mock_container('web', 1), + mock_container('web', 2), + mock_container('db', 1), + mock_container('other', 1), + mock_container('another', 1), + ] + service_names = ['web', 'db'] + log_printer = build_log_printer(containers, service_names, True) + self.assertEqual(log_printer.containers, containers[:3]) + + def test_attach_to_logs(self): + project = mock.create_autospec(Project) + log_printer = mock.create_autospec(LogPrinter, containers=[]) + service_names = ['web', 'db'] + timeout = 12 + + with mock.patch('compose.cli.main.signal', autospec=True) as mock_signal: + attach_to_logs(project, log_printer, service_names, timeout) + + mock_signal.signal.assert_called_once_with(mock_signal.SIGINT, mock.ANY) + log_printer.run.assert_called_once_with() + project.stop.assert_called_once_with( + service_names=service_names, + timeout=timeout) From acca22206ab7a3eeb1cad99d2a93f4e9190e67ff Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 27 Aug 2015 10:36:12 +0100 Subject: [PATCH 094/337] Fix typo Signed-off-by: Aanand Prasad --- RELEASE_PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 0d5f42eba0..966e06ee48 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -93,7 +93,7 @@ Building a Compose release Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. - Secondly, if you're a Mac user, the **[Docker Toolbox](https://www.docker.com/toolbox)** will install Compose ${VERSION} for you, alongside the latest versions of the Docker Engine, Machine and Kitematic. + Secondly, if you're a Mac user, the **[Docker Toolbox](https://www.docker.com/toolbox)** will install Compose 1.5.0 for you, alongside the latest versions of the Docker Engine, Machine and Kitematic. Otherwise, you can use the usual commands to install/upgrade. Either download the binary: From b39e549c87696d7b1d3ee10ebb1880bd791c647f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 27 Aug 2015 13:35:45 +0100 Subject: [PATCH 095/337] Normalise ignore files - Consistent order and contents (where possible) - Prepend .gitignore paths with slashes where appropriate Signed-off-by: Aanand Prasad --- .dockerignore | 9 +++++++-- .gitignore | 7 ++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index b85b7e5d86..ba7e9155d5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,10 @@ +*.egg-info +.coverage .git +.tox build -dist -venv coverage-html +dist +docker-compose.spec +docs/_site +venv diff --git a/.gitignore b/.gitignore index 52a78bd974..f6750c1ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ *.egg-info *.pyc -.tox +/.coverage +/.tox /build +/coverage-html /dist +/docker-compose.spec /docs/_site /venv -docker-compose.spec -coverage-html From 477d4f491dca36f9c717f8bb6366d1f756a387bc Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Wed, 26 Aug 2015 22:03:45 +0100 Subject: [PATCH 096/337] Do not allow to specify both image and dockerfile in configuration. Closes #1908 Signed-off-by: Karol Duleba --- compose/config/schema.json | 5 ++++- compose/config/validation.py | 5 +++++ docs/yml.md | 6 ++++++ tests/unit/config_test.py | 11 +++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/compose/config/schema.json b/compose/config/schema.json index 8e9b79fb64..94fe4fc522 100644 --- a/compose/config/schema.json +++ b/compose/config/schema.json @@ -113,7 +113,10 @@ }, { "required": ["image"], - "not": {"required": ["build"]} + "not": {"anyOf": [ + {"required": ["build"]}, + {"required": ["dockerfile"]} + ]} }, { "required": ["extends"], diff --git a/compose/config/validation.py b/compose/config/validation.py index 0df73e3c25..d83504274c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -134,6 +134,11 @@ def process_errors(errors): required.append( "Service '{}' has neither an image nor a build path " "specified. Exactly one must be provided.".format(service_name)) + elif 'image' in error.instance and 'dockerfile' in error.instance: + required.append( + "Service '{}' has both an image and alternate Dockerfile. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) else: required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': diff --git a/docs/yml.md b/docs/yml.md index 6fb31a7db9..3ece026494 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -33,6 +33,8 @@ pull if it doesn't exist locally. image: a4bc65fd image: busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d +Using `image` together with either `build` or `dockerfile` is not allowed. Attempting to do so results in an error. + ### build Path to a directory containing a Dockerfile. When the value supplied is a @@ -43,6 +45,8 @@ Compose will build and tag it with a generated name, and use that image thereaft build: /path/to/build/dir +Using `build` together with `image` is not allowed. Attempting to do so results in an error. + ### dockerfile Alternate Dockerfile. @@ -51,6 +55,8 @@ Compose will use an alternate file to build with. dockerfile: Dockerfile-alternate +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + ### command Override the default command. diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 51dac052f3..e488ceb527 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -191,6 +191,17 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_image_and_dockerfile_raise_validation_error(self): + expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, + 'working_dir', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From d264c2e33a57469e17f97ff06819b3203f81a4b1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 27 Aug 2015 17:42:26 -0400 Subject: [PATCH 097/337] Resolves #1804 Fix mutation of service.options when a label or environment variable is specified in the config. Signed-off-by: Daniel Nephin --- compose/config/config.py | 2 +- compose/service.py | 20 +++++++++----------- tests/unit/service_test.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ea122bc422..cfa8086f09 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -358,7 +358,7 @@ def parse_environment(environment): return dict(split_env(e) for e in environment) if isinstance(environment, dict): - return environment + return dict(environment) raise ConfigurationError( "environment \"%s\" must be a list or mapping," % diff --git a/compose/service.py b/compose/service.py index a0423ff447..b48f2e14bd 100644 --- a/compose/service.py +++ b/compose/service.py @@ -576,7 +576,6 @@ class Service(object): number, one_off=False, previous_container=None): - add_config_hash = (not one_off and not override_options) container_options = dict( @@ -589,13 +588,6 @@ class Service(object): elif not container_options.get('name'): container_options['name'] = self.get_container_name(number, one_off) - if add_config_hash: - config_hash = self.config_hash - if 'labels' not in container_options: - container_options['labels'] = {} - container_options['labels'][LABEL_CONFIG_HASH] = config_hash - log.debug("Added config hash: %s" % config_hash) - if 'detach' not in container_options: container_options['detach'] = True @@ -643,7 +635,8 @@ class Service(object): container_options['labels'] = build_container_labels( container_options.get('labels', {}), self.labels(one_off=one_off), - number) + number, + self.config_hash if add_config_hash else None) # Delete options which are only used when starting for key in DOCKER_START_KEYS: @@ -899,11 +892,16 @@ def parse_volume_spec(volume_config): # Labels -def build_container_labels(label_options, service_labels, number, one_off=False): - labels = label_options or {} +def build_container_labels(label_options, service_labels, number, config_hash): + labels = dict(label_options or {}) labels.update(label.split('=', 1) for label in service_labels) labels[LABEL_CONTAINER_NUMBER] = str(number) labels[LABEL_VERSION] = __version__ + + if config_hash: + log.debug("Added config hash: %s" % config_hash) + labels[LABEL_CONFIG_HASH] = config_hash + return labels diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a24e524dd5..aa6d4d74f4 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -6,6 +6,7 @@ from docker.utils import LogConfig from .. import mock from .. import unittest +from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE @@ -163,6 +164,40 @@ class ServiceTest(unittest.TestCase): one_off=True) self.assertEqual(opts['name'], name) + def test_get_container_create_options_does_not_mutate_options(self): + labels = {'thing': 'real'} + environment = {'also': 'real'} + service = Service( + 'foo', + image='foo', + labels=dict(labels), + client=self.mock_client, + environment=dict(environment), + ) + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + prev_container = mock.Mock( + id='ababab', + image_config={'ContainerConfig': {}}) + + opts = service._get_container_create_options( + {}, + 1, + previous_container=prev_container) + + self.assertEqual(service.options['labels'], labels) + self.assertEqual(service.options['environment'], environment) + + self.assertEqual( + opts['labels'][LABEL_CONFIG_HASH], + 'b30306d0a73b67f67a45b99b88d36c359e470e6fa0c04dda1cf62d2087205b81') + self.assertEqual( + opts['environment'], + { + 'affinity:container': '=ababab', + 'also': 'real', + } + ) + def test_get_container_not_found(self): self.mock_client.containers.return_value = [] service = Service('foo', client=self.mock_client, image='foo') From 9543cb341bc98b7ef7a61cbb2d2543e446dea163 Mon Sep 17 00:00:00 2001 From: Herman Junge Date: Thu, 27 Aug 2015 19:41:17 -0300 Subject: [PATCH 098/337] Fix doc install.md termial -> terminal Signed-off-by: Herman Junge --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 7a2763edc0..85060ce040 100644 --- a/docs/install.md +++ b/docs/install.md @@ -35,7 +35,7 @@ To install Compose, do the following: 3. Go to the repository release page. -4. Enter the `curl` command in your termial. +4. Enter the `curl` command in your terminal. The command has the following format: From a4bab13aee9b5804e74e6192bc412fccbdf6d8d5 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Fri, 28 Aug 2015 19:05:19 +0200 Subject: [PATCH 099/337] Adds pause- and unpause-command to docopt's TLC solves # 1921 Signed-off-by: Frank Sachsenheim --- compose/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2ace13c22b..06dacf1e9a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -101,6 +101,7 @@ class TopLevelCommand(Command): help Get help on a command kill Kill containers logs View output from containers + pause Pause services port Print the public port for a port binding ps List containers pull Pulls service images @@ -110,6 +111,7 @@ class TopLevelCommand(Command): scale Set number of containers for a service start Start services stop Stop services + unpause Unpause services up Create and start containers migrate-to-labels Recreate containers to add labels version Show the Docker-Compose version information From 8f310767a63c4cdb151593cb5dd2e8808516ba4f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 28 Aug 2015 14:01:02 -0400 Subject: [PATCH 100/337] Add ISSUE-TRIAGE.md doc Signed-off-by: Daniel Nephin --- project/ISSUE-TRIAGE.md | 32 +++++++++++++++++++ .../RELEASE-PROCESS.md | 0 2 files changed, 32 insertions(+) create mode 100644 project/ISSUE-TRIAGE.md rename RELEASE_PROCESS.md => project/RELEASE-PROCESS.md (100%) diff --git a/project/ISSUE-TRIAGE.md b/project/ISSUE-TRIAGE.md new file mode 100644 index 0000000000..bcedbb4359 --- /dev/null +++ b/project/ISSUE-TRIAGE.md @@ -0,0 +1,32 @@ +Triaging of issues +------------------ + +The docker-compose issue triage process follows +https://github.com/docker/docker/blob/master/project/ISSUE-TRIAGE.md +with the following additions or exceptions. + + +### Classify the Issue + +The following labels are provided in additional to the standard labels: + +| Kind | Description | +|--------------|-------------------------------------------------------------------| +| kind/cleanup | A refactor or improvement that is related to quality not function | +| kind/parity | A request for feature parity with docker cli | + + +### Functional areas + +Most issues should fit into one of the following functional areas: + +| Area | +|-------------| +| area/build | +| area/cli | +| area/config | +| area/logs | +| area/run | +| area/scale | +| area/tests | +| area/up | diff --git a/RELEASE_PROCESS.md b/project/RELEASE-PROCESS.md similarity index 100% rename from RELEASE_PROCESS.md rename to project/RELEASE-PROCESS.md From 235fe21fd0ad3097e6e35692bc2f25b1c2062bc9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 26 Aug 2015 18:55:22 -0400 Subject: [PATCH 101/337] Prevent flaky test by changing container names. Signed-off-by: Daniel Nephin --- tests/integration/service_test.py | 2 +- tests/integration/state_test.py | 1 - tests/integration/testcases.py | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index fc634c8ca3..0cf8cdb0ef 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -695,7 +695,7 @@ class ServiceTest(DockerClientTestCase): Test that calling scale on a service that has a custom container name results in warning output. """ - service = self.create_service('web', container_name='custom-container') + service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name(), 'custom-container') service.scale(3) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 3d4a5b5aa6..b3dd42d996 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -199,7 +199,6 @@ class ServiceStateTest(DockerClientTestCase): self.assertEqual([c.is_running for c in containers], [False, True]) - web = self.create_service('web', **options) self.assertEqual( ('start', containers[0:1]), web.convergence_plan(), diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index e239010ea2..08ef9f272f 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -33,6 +33,9 @@ class DockerClientTestCase(unittest.TestCase): options = ServiceLoader(working_dir='.').make_service_dict(name, kwargs) + labels = options.setdefault('labels', {}) + labels['com.docker.compose.test-name'] = self.id() + return Service( project='composetest', client=self.client, From 5a5f28228a44975899a1b2787ac4b9ad72b74bea Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 25 Aug 2015 10:53:31 -0400 Subject: [PATCH 102/337] Document installing of pre-commit hooks. Signed-off-by: Daniel Nephin --- CONTRIBUTING.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9188ac98a..9ff8304c76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,17 @@ that should get you started. `docker-compose` from anywhere on your machine, it will run your development version of Compose. +## Install pre-commit hooks + +This step is optional, but recommended. Pre-commit hooks will run style checks +and in some cases fix style issues for you, when you commit code. + +Install the git pre-commit hooks using [tox](https://tox.readthedocs.org) by +running `tox -e pre-commit` or by following the +[pre-commit install guide](http://pre-commit.com/#install). + +To run the style checks at any time run `tox -e pre-commit`. + ## Submitting a pull request See Docker's [basic contribution workflow](https://docs.docker.com/project/make-a-contribution/#the-basic-contribution-workflow) for a guide on how to submit a pull request for code or documentation. From 74782a56b53638431c93f0081e8d933f7fc0a104 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 25 Aug 2015 10:57:21 -0400 Subject: [PATCH 103/337] Fix script/test by just calling script/test-versions directly instead of launching another container. Signed-off-by: Daniel Nephin --- script/ci | 1 + script/test | 15 +++++---------- script/test-versions | 5 ++++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/script/ci b/script/ci index b497548781..e8356bb73f 100755 --- a/script/ci +++ b/script/ci @@ -10,6 +10,7 @@ set -e export DOCKER_VERSIONS=all export DOCKER_DAEMON_ARGS="--storage-driver=overlay" +GIT_VOLUME="--volumes-from=$(hostname)" . script/test-versions >&2 echo "Building Linux binary" diff --git a/script/test b/script/test index adf3fb1bab..bdb3579b01 100755 --- a/script/test +++ b/script/test @@ -6,15 +6,10 @@ set -ex TAG="docker-compose:$(git rev-parse --short HEAD)" rm -rf coverage-html +# Create the host directory so it's owned by $USER +mkdir -p coverage-html docker build -t "$TAG" . -docker run \ - --rm \ - --volume="/var/run/docker.sock:/var/run/docker.sock" \ - -e DOCKER_VERSIONS \ - -e "TAG=$TAG" \ - -e "affinity:image==$TAG" \ - -e "COVERAGE_DIR=$(pwd)/coverage-html" \ - --entrypoint="script/test-versions" \ - "$TAG" \ - "$@" + +GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" +. script/test-versions diff --git a/script/test-versions b/script/test-versions index f39c17e819..88d2554c2b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -5,7 +5,10 @@ set -e >&2 echo "Running lint checks" -tox -e pre-commit +docker run --rm \ + ${GIT_VOLUME} \ + --entrypoint="tox" \ + "$TAG" -e pre-commit if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="default" From 6ac617bae1871dc6dcd53c1108376f2a920541a0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 16:23:04 -0400 Subject: [PATCH 104/337] Split requirements-build.txt from requirements-dev.txt to support a leaner tox.ini Signed-off-by: Daniel Nephin --- Dockerfile | 2 +- requirements-build.txt | 1 + requirements-dev.txt | 1 - script/build-linux | 1 - script/build-linux-inner | 9 ++++++--- script/build-osx | 2 +- script/ci | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 requirements-build.txt diff --git a/Dockerfile b/Dockerfile index a9892031b8..1d13c2b603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,7 +91,7 @@ RUN pip install -r requirements-dev.txt RUN pip install tox==2.1.1 ADD . /code/ -RUN python setup.py install +RUN pip install --no-deps -e /code RUN chown -R user /code/ diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 0000000000..5da6fa4966 --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1 @@ +git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller diff --git a/requirements-dev.txt b/requirements-dev.txt index adb4387df2..33a4915154 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ coverage==3.7.1 flake8==2.3.0 -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller mock >= 1.0.1 nose==1.3.4 pep8==1.6.1 diff --git a/script/build-linux b/script/build-linux index 5e4a9470e9..4fdf1d926f 100755 --- a/script/build-linux +++ b/script/build-linux @@ -6,7 +6,6 @@ TAG="docker-compose" docker build -t "$TAG" . docker run \ --rm \ - --user=user \ --volume="$(pwd):/code" \ --entrypoint="script/build-linux-inner" \ "$TAG" diff --git a/script/build-linux-inner b/script/build-linux-inner index adc030eaa8..cfea838067 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -2,9 +2,12 @@ set -ex +TARGET=dist/docker-compose-Linux-x86_64 + mkdir -p `pwd`/dist chmod 777 `pwd`/dist -pyinstaller -F bin/docker-compose -mv dist/docker-compose dist/docker-compose-Linux-x86_64 -dist/docker-compose-Linux-x86_64 version +pip install -r requirements-build.txt +su -c "pyinstaller -F bin/docker-compose" user +mv dist/docker-compose $TARGET +$TARGET version diff --git a/script/build-osx b/script/build-osx index 2a9cf512ef..d99c1fb981 100755 --- a/script/build-osx +++ b/script/build-osx @@ -6,7 +6,7 @@ PATH="/usr/local/bin:$PATH" rm -rf venv virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt -venv/bin/pip install -r requirements-dev.txt +venv/bin/pip install -r requirements-build.txt venv/bin/pip install . venv/bin/pyinstaller -F bin/docker-compose mv dist/docker-compose dist/docker-compose-Darwin-x86_64 diff --git a/script/ci b/script/ci index b497548781..e392fae759 100755 --- a/script/ci +++ b/script/ci @@ -13,4 +13,4 @@ export DOCKER_DAEMON_ARGS="--storage-driver=overlay" . script/test-versions >&2 echo "Building Linux binary" -su -c script/build-linux-inner user +. script/build-linux-inner From c1ed1efde81dc2c93b5231cd67416fb89091377e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 16:24:07 -0400 Subject: [PATCH 105/337] Use py.test as the test runner Signed-off-by: Daniel Nephin --- requirements-dev.txt | 7 +++---- setup.py | 5 +---- tox.ini | 28 ++++++++++++++++------------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 33a4915154..73b8078350 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ coverage==3.7.1 -flake8==2.3.0 -mock >= 1.0.1 -nose==1.3.4 -pep8==1.6.1 +mock>=1.0.1 +pytest==2.7.2 +pytest-cov==2.1.0 diff --git a/setup.py b/setup.py index 33335047bc..e93dafc628 100644 --- a/setup.py +++ b/setup.py @@ -41,13 +41,10 @@ install_requires = [ tests_require = [ - 'nose', - 'flake8', + 'pytest', ] -if sys.version_info < (2, 7): - tests_require.append('unittest2') if sys.version_info[:1] < (3,): tests_require.append('mock >= 1.0.1') diff --git a/tox.ini b/tox.ini index 4b27a4e9be..71ab4fc9c9 100644 --- a/tox.ini +++ b/tox.ini @@ -8,10 +8,14 @@ passenv = setenv = HOME=/tmp deps = - -rrequirements.txt + -rrequirements-dev.txt commands = - nosetests -v --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html {posargs} - flake8 compose tests setup.py + py.test -v \ + --cov=compose \ + --cov-report html \ + --cov-report term \ + --cov-config=tox.ini \ + {posargs} [testenv:pre-commit] skip_install = True @@ -21,16 +25,16 @@ commands = pre-commit install pre-commit run --all-files -[testenv:py27] -deps = - {[testenv]deps} - -rrequirements-dev.txt +# Coverage configuration +[run] +branch = True -[testenv:py34] -deps = - {[testenv]deps} - flake8 - nose +[report] +show_missing = true + +[html] +directory = coverage-html +# end coverage configuration [flake8] # Allow really long lines for now From 6969829a705fe784213b139f7087708fd0b026b5 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 1 Sep 2015 17:23:39 -0700 Subject: [PATCH 106/337] Link to ZenHub instead of Waffle Signed-off-by: Ben Firshman --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9188ac98a..cb26a5501a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,6 @@ you can specify a test directory, file, module, class or method: ## Finding things to work on -We use a [Waffle.io board](https://waffle.io/docker/compose) to keep track of specific things we are working on and planning to work on. If you're looking for things to work on, stuff in the backlog is a great place to start. +We use a [ZenHub board](https://www.zenhub.io/) to keep track of specific things we are working on and planning to work on. If you're looking for things to work on, stuff in the backlog is a great place to start. For more information about our project planning, take a look at our [GitHub wiki](https://github.com/docker/compose/wiki). From c907f35e741ff2f40c775c07d311f53a0d4c2373 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 20 Aug 2015 15:05:33 +0100 Subject: [PATCH 107/337] Raise if working_dir is None Check for this in the init so we can remove the duplication of raising in further functions. A ServiceLoader isn't valid without one. Signed-off-by: Mazz Mosley --- compose/config/config.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index cfa8086f09..e08b503f3e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -152,7 +152,11 @@ def load(config_details): class ServiceLoader(object): def __init__(self, working_dir, filename=None, already_seen=None): + if working_dir is None: + raise Exception("No working_dir passed to ServiceLoader()") + self.working_dir = os.path.abspath(working_dir) + if filename: self.filename = os.path.abspath(filename) else: @@ -176,9 +180,6 @@ class ServiceLoader(object): extends_options = self.validate_extends_options(service_dict['name'], service_dict['extends']) - if self.working_dir is None: - raise Exception("No working_dir passed to ServiceLoader()") - if 'file' in extends_options: extends_from_filename = extends_options['file'] other_config_path = expand_path(self.working_dir, extends_from_filename) @@ -320,9 +321,6 @@ def get_env_files(options, working_dir=None): if 'env_file' not in options: return {} - if working_dir is None: - raise Exception("No working_dir passed to get_env_files()") - env_files = options.get('env_file', []) if not isinstance(env_files, list): env_files = [env_files] From 1344533b240ddc344029536df8361125617e1a3d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 21 Aug 2015 17:04:10 +0100 Subject: [PATCH 108/337] filename is not optional While it can be set to ultimately a value of None, when a config file is read in from stdin, it is not optional. We kinda make use of it's ability to be set to None in our tests but functionally and design wise, it is required. If filename is not set, extends does not work. Signed-off-by: Mazz Mosley --- compose/config/config.py | 2 +- tests/integration/testcases.py | 2 +- tests/unit/config_test.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index e08b503f3e..b7697f0020 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -151,7 +151,7 @@ def load(config_details): class ServiceLoader(object): - def __init__(self, working_dir, filename=None, already_seen=None): + def __init__(self, working_dir, filename, already_seen=None): if working_dir is None: raise Exception("No working_dir passed to ServiceLoader()") diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 08ef9f272f..d9d666d270 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -31,7 +31,7 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - options = ServiceLoader(working_dir='.').make_service_dict(name, kwargs) + options = ServiceLoader(working_dir='.', filename=None).make_service_dict(name, kwargs) labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index e488ceb527..aa10982b46 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -11,11 +11,11 @@ from compose.config import config from compose.config.errors import ConfigurationError -def make_service_dict(name, service_dict, working_dir): +def make_service_dict(name, service_dict, working_dir, filename=None): """ Test helper function to construct a ServiceLoader """ - return config.ServiceLoader(working_dir=working_dir).make_service_dict(name, service_dict) + return config.ServiceLoader(working_dir=working_dir, filename=filename).make_service_dict(name, service_dict) def service_sort(services): From 8a6061bfb9a3e4a98c11ad385eee45710af81e3f Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 21 Aug 2015 17:07:37 +0100 Subject: [PATCH 109/337] __init__ takes service name and dict Moving service name and dict out of the function make_service_dict and into __init__. We always call make_service_dict with those so let's put them in the initialiser. Slightly cleaner design intent. The whole purpose of the ServiceLoader is to take a service name&service dictionary then validate, process and return service dictionaries ready to be created. This is also another step towards cleaning the code up so we can interpolate and validate an extended dictionary. Signed-off-by: Mazz Mosley --- compose/config/config.py | 42 +++++++++++++++++++--------------- tests/integration/testcases.py | 2 +- tests/unit/config_test.py | 6 ++++- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index b7697f0020..9d90bd6142 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -142,8 +142,12 @@ def load(config_details): service_dicts = [] for service_name, service_dict in list(processed_config.items()): - loader = ServiceLoader(working_dir=working_dir, filename=filename) - service_dict = loader.make_service_dict(service_name, service_dict) + loader = ServiceLoader( + working_dir=working_dir, + filename=filename, + service_name=service_name, + service_dict=service_dict) + service_dict = loader.make_service_dict() validate_paths(service_dict) service_dicts.append(service_dict) @@ -151,7 +155,7 @@ def load(config_details): class ServiceLoader(object): - def __init__(self, working_dir, filename, already_seen=None): + def __init__(self, working_dir, filename, service_name, service_dict, already_seen=None): if working_dir is None: raise Exception("No working_dir passed to ServiceLoader()") @@ -162,17 +166,19 @@ class ServiceLoader(object): else: self.filename = filename self.already_seen = already_seen or [] + self.service_dict = service_dict.copy() + self.service_dict['name'] = service_name def detect_cycle(self, name): if self.signature(name) in self.already_seen: raise CircularReference(self.already_seen + [self.signature(name)]) - def make_service_dict(self, name, service_dict): - service_dict = service_dict.copy() - service_dict['name'] = name - service_dict = resolve_environment(service_dict, working_dir=self.working_dir) - service_dict = self.resolve_extends(service_dict) - return process_container_options(service_dict, working_dir=self.working_dir) + def make_service_dict(self): + # service_dict = service_dict.copy() + # service_dict['name'] = name + self.service_dict = resolve_environment(self.service_dict, working_dir=self.working_dir) + self.service_dict = self.resolve_extends(self.service_dict) + return process_container_options(self.service_dict, working_dir=self.working_dir) def resolve_extends(self, service_dict): if 'extends' not in service_dict: @@ -188,11 +194,6 @@ class ServiceLoader(object): other_working_dir = os.path.dirname(other_config_path) other_already_seen = self.already_seen + [self.signature(service_dict['name'])] - other_loader = ServiceLoader( - working_dir=other_working_dir, - filename=other_config_path, - already_seen=other_already_seen, - ) base_service = extends_options['service'] other_config = load_yaml(other_config_path) @@ -204,11 +205,16 @@ class ServiceLoader(object): raise ConfigurationError(msg) other_service_dict = other_config[base_service] - other_loader.detect_cycle(extends_options['service']) - other_service_dict = other_loader.make_service_dict( - service_dict['name'], - other_service_dict, + other_loader = ServiceLoader( + working_dir=other_working_dir, + filename=other_config_path, + service_name=service_dict['name'], + service_dict=other_service_dict, + already_seen=other_already_seen, ) + + other_loader.detect_cycle(extends_options['service']) + other_service_dict = other_loader.make_service_dict() validate_extended_service_dict( other_service_dict, filename=other_config_path, diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index d9d666d270..58240d5ea4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -31,7 +31,7 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - options = ServiceLoader(working_dir='.', filename=None).make_service_dict(name, kwargs) + options = ServiceLoader(working_dir='.', filename=None, service_name=name, service_dict=kwargs).make_service_dict() labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index aa10982b46..f3a4bd306a 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -15,7 +15,11 @@ def make_service_dict(name, service_dict, working_dir, filename=None): """ Test helper function to construct a ServiceLoader """ - return config.ServiceLoader(working_dir=working_dir, filename=filename).make_service_dict(name, service_dict) + return config.ServiceLoader( + working_dir=working_dir, + filename=filename, + service_name=name, + service_dict=service_dict).make_service_dict() def service_sort(services): From 02c52ae673a66c7a8f6455611d8561d8f6954383 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 24 Aug 2015 12:23:45 +0100 Subject: [PATCH 110/337] Move resolve_environment within __init__ resolve_environment is specific to ServiceLoader, the function does not need to be on the global scope, it is a part of the ServiceLoader object. The environment needs to be resolved before we can make any service dicts, it belongs in the constructor. This is cleaning up the design a little and being clearer about intent and scope of functions. Signed-off-by: Mazz Mosley --- compose/config/config.py | 45 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9d90bd6142..ff9b359333 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -169,17 +169,36 @@ class ServiceLoader(object): self.service_dict = service_dict.copy() self.service_dict['name'] = service_name + self.resolve_environment() + def detect_cycle(self, name): if self.signature(name) in self.already_seen: raise CircularReference(self.already_seen + [self.signature(name)]) def make_service_dict(self): - # service_dict = service_dict.copy() - # service_dict['name'] = name - self.service_dict = resolve_environment(self.service_dict, working_dir=self.working_dir) self.service_dict = self.resolve_extends(self.service_dict) return process_container_options(self.service_dict, working_dir=self.working_dir) + def resolve_environment(self): + """ + Unpack any environment variables from an env_file, if set. + Interpolate environment values if set. + """ + if 'environment' not in self.service_dict and 'env_file' not in self.service_dict: + return + + env = {} + + if 'env_file' in self.service_dict: + for f in get_env_files(self.service_dict, working_dir=self.working_dir): + env.update(env_vars_from_file(f)) + del self.service_dict['env_file'] + + env.update(parse_environment(self.service_dict.get('environment'))) + env = dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) + + self.service_dict['environment'] = env + def resolve_extends(self, service_dict): if 'extends' not in service_dict: return service_dict @@ -334,26 +353,6 @@ def get_env_files(options, working_dir=None): return [expand_path(working_dir, path) for path in env_files] -def resolve_environment(service_dict, working_dir=None): - service_dict = service_dict.copy() - - if 'environment' not in service_dict and 'env_file' not in service_dict: - return service_dict - - env = {} - - if 'env_file' in service_dict: - for f in get_env_files(service_dict, working_dir=working_dir): - env.update(env_vars_from_file(f)) - del service_dict['env_file'] - - env.update(parse_environment(service_dict.get('environment'))) - env = dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) - - service_dict['environment'] = env - return service_dict - - def parse_environment(environment): if not environment: return {} From 538a501eece5f645285f8235cf21507127750300 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 24 Aug 2015 15:58:17 +0100 Subject: [PATCH 111/337] Refactor validating extends file path Separating out the steps we need to resolve extends, so that it will be clear to insert pre-processing of interpolation and validation. Signed-off-by: Mazz Mosley --- compose/config/config.py | 36 ++++++++++++++++++------------------ compose/config/validation.py | 13 +++++++++++++ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ff9b359333..51bd938466 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -11,6 +11,7 @@ from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables from .validation import validate_against_schema +from .validation import validate_extends_file_path from .validation import validate_service_names from .validation import validate_top_level_object from compose.cli.utils import find_candidates_in_parent_dirs @@ -171,12 +172,20 @@ class ServiceLoader(object): self.resolve_environment() + if 'extends' in self.service_dict: + validate_extends_file_path( + service_name, + self.service_dict['extends'], + self.filename + ) + + def detect_cycle(self, name): if self.signature(name) in self.already_seen: raise CircularReference(self.already_seen + [self.signature(name)]) def make_service_dict(self): - self.service_dict = self.resolve_extends(self.service_dict) + self.service_dict = self.resolve_extends() return process_container_options(self.service_dict, working_dir=self.working_dir) def resolve_environment(self): @@ -199,11 +208,12 @@ class ServiceLoader(object): self.service_dict['environment'] = env - def resolve_extends(self, service_dict): - if 'extends' not in service_dict: - return service_dict + def resolve_extends(self): + if 'extends' not in self.service_dict: + return self.service_dict - extends_options = self.validate_extends_options(service_dict['name'], service_dict['extends']) + extends_options = self.service_dict['extends'] + service_name = self.service_dict['name'] if 'file' in extends_options: extends_from_filename = extends_options['file'] @@ -212,7 +222,7 @@ class ServiceLoader(object): other_config_path = self.filename other_working_dir = os.path.dirname(other_config_path) - other_already_seen = self.already_seen + [self.signature(service_dict['name'])] + other_already_seen = self.already_seen + [self.signature(service_name)] base_service = extends_options['service'] other_config = load_yaml(other_config_path) @@ -227,7 +237,7 @@ class ServiceLoader(object): other_loader = ServiceLoader( working_dir=other_working_dir, filename=other_config_path, - service_name=service_dict['name'], + service_name=service_name, service_dict=other_service_dict, already_seen=other_already_seen, ) @@ -240,21 +250,11 @@ class ServiceLoader(object): service=extends_options['service'], ) - return merge_service_dicts(other_service_dict, service_dict) + return merge_service_dicts(other_service_dict, self.service_dict) def signature(self, name): return (self.filename, name) - def validate_extends_options(self, service_name, extends_options): - error_prefix = "Invalid 'extends' configuration for %s:" % service_name - - if 'file' not in extends_options and self.filename is None: - raise ConfigurationError( - "%s you need to specify a 'file', e.g. 'file: something.yml'" % error_prefix - ) - - return extends_options - def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) diff --git a/compose/config/validation.py b/compose/config/validation.py index d83504274c..1ae8981ca6 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -66,6 +66,19 @@ def validate_top_level_object(func): return func_wrapper +def validate_extends_file_path(service_name, extends_options, filename): + """ + The service to be extended must either be defined in the config key 'file', + or within 'filename'. + """ + error_prefix = "Invalid 'extends' configuration for %s:" % service_name + + if 'file' not in extends_options and filename is None: + raise ConfigurationError( + "%s you need to specify a 'file', e.g. 'file: something.yml'" % error_prefix + ) + + def get_unsupported_config_msg(service_name, error_key): msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) if error_key in DOCKER_CONFIG_HINTS: From 37bf8235b71a45b4b303b937129e07997784c61b Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 24 Aug 2015 16:00:47 +0100 Subject: [PATCH 112/337] Get extended config path Refactored out into it's own function. Signed-off-by: Mazz Mosley --- compose/config/config.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 51bd938466..0f3099dc07 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -179,6 +179,9 @@ class ServiceLoader(object): self.filename ) + self.extended_config_path = self.get_extended_config_path( + self.service_dict['extends'] + ) def detect_cycle(self, name): if self.signature(name) in self.already_seen: @@ -215,11 +218,7 @@ class ServiceLoader(object): extends_options = self.service_dict['extends'] service_name = self.service_dict['name'] - if 'file' in extends_options: - extends_from_filename = extends_options['file'] - other_config_path = expand_path(self.working_dir, extends_from_filename) - else: - other_config_path = self.filename + other_config_path = self.get_extended_config_path(extends_options) other_working_dir = os.path.dirname(other_config_path) other_already_seen = self.already_seen + [self.signature(service_name)] @@ -252,6 +251,18 @@ class ServiceLoader(object): return merge_service_dicts(other_service_dict, self.service_dict) + def get_extended_config_path(self, extends_options): + """ + Service we are extending either has a value for 'file' set, which we + need to obtain a full path too or we are extending from a service + defined in our own file. + """ + if 'file' in extends_options: + extends_from_filename = extends_options['file'] + return expand_path(self.working_dir, extends_from_filename) + + return self.filename + def signature(self, name): return (self.filename, name) From 36757cde1cbe38d9673f00af0f515038b8280cfe Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 25 Aug 2015 17:54:06 +0100 Subject: [PATCH 113/337] Validate extended service against our schema Signed-off-by: Mazz Mosley --- compose/config/config.py | 7 +-- tests/fixtures/extends/invalid-links.yml | 9 ++++ tests/fixtures/extends/invalid-net.yml | 8 ++++ tests/fixtures/extends/invalid-volumes.yml | 9 ++++ .../extends/service-with-invalid-schema.yml | 5 +++ tests/unit/config_test.py | 45 ++++++++----------- 6 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 tests/fixtures/extends/invalid-links.yml create mode 100644 tests/fixtures/extends/invalid-net.yml create mode 100644 tests/fixtures/extends/invalid-volumes.yml create mode 100644 tests/fixtures/extends/service-with-invalid-schema.yml diff --git a/compose/config/config.py b/compose/config/config.py index 0f3099dc07..65a5b5472d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -182,6 +182,8 @@ class ServiceLoader(object): self.extended_config_path = self.get_extended_config_path( self.service_dict['extends'] ) + extended_config = load_yaml(self.extended_config_path) + validate_against_schema(extended_config) def detect_cycle(self, name): if self.signature(name) in self.already_seen: @@ -217,10 +219,9 @@ class ServiceLoader(object): extends_options = self.service_dict['extends'] service_name = self.service_dict['name'] + other_config_path = self.extended_config_path - other_config_path = self.get_extended_config_path(extends_options) - - other_working_dir = os.path.dirname(other_config_path) + other_working_dir = os.path.dirname(self.extended_config_path) other_already_seen = self.already_seen + [self.signature(service_name)] base_service = extends_options['service'] diff --git a/tests/fixtures/extends/invalid-links.yml b/tests/fixtures/extends/invalid-links.yml new file mode 100644 index 0000000000..edfeb8b231 --- /dev/null +++ b/tests/fixtures/extends/invalid-links.yml @@ -0,0 +1,9 @@ +myweb: + build: '.' + extends: + service: web + command: top +web: + build: '.' + links: + - "mydb:db" diff --git a/tests/fixtures/extends/invalid-net.yml b/tests/fixtures/extends/invalid-net.yml new file mode 100644 index 0000000000..fbcd020bcf --- /dev/null +++ b/tests/fixtures/extends/invalid-net.yml @@ -0,0 +1,8 @@ +myweb: + build: '.' + extends: + service: web + command: top +web: + build: '.' + net: "container:db" diff --git a/tests/fixtures/extends/invalid-volumes.yml b/tests/fixtures/extends/invalid-volumes.yml new file mode 100644 index 0000000000..3db0118e0e --- /dev/null +++ b/tests/fixtures/extends/invalid-volumes.yml @@ -0,0 +1,9 @@ +myweb: + build: '.' + extends: + service: web + command: top +web: + build: '.' + volumes_from: + - "db" diff --git a/tests/fixtures/extends/service-with-invalid-schema.yml b/tests/fixtures/extends/service-with-invalid-schema.yml new file mode 100644 index 0000000000..90dc76a0ea --- /dev/null +++ b/tests/fixtures/extends/service-with-invalid-schema.yml @@ -0,0 +1,5 @@ +myweb: + extends: + service: web +web: + command: top diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f3a4bd306a..98ae513854 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -867,6 +867,12 @@ class ExtendsTest(unittest.TestCase): self.assertEquals(len(service), 1) self.assertIsInstance(service[0], dict) + def test_extended_service_with_invalid_config(self): + expected_error_msg = "Service 'myweb' has neither an image nor a build path specified" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + load_from_filename('tests/fixtures/extends/service-with-invalid-schema.yml') + def test_extends_file_defaults_to_self(self): """ Test not specifying a file in our extends options that the @@ -891,37 +897,22 @@ class ExtendsTest(unittest.TestCase): } ])) - def test_blacklisted_options(self): - def load_config(): - return make_service_dict('myweb', { - 'extends': { - 'file': 'whatever', - 'service': 'web', - } - }, '.') + def test_invalid_links_in_extended_service(self): + expected_error_msg = "services with 'links' cannot be extended" + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + load_from_filename('tests/fixtures/extends/invalid-links.yml') - with self.assertRaisesRegexp(ConfigurationError, 'links'): - other_config = {'web': {'links': ['db']}} + def test_invalid_volumes_from_in_extended_service(self): + expected_error_msg = "services with 'volumes_from' cannot be extended" - with mock.patch.object(config, 'load_yaml', return_value=other_config): - print(load_config()) + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + load_from_filename('tests/fixtures/extends/invalid-volumes.yml') - with self.assertRaisesRegexp(ConfigurationError, 'volumes_from'): - other_config = {'web': {'volumes_from': ['db']}} + def test_invalid_net_in_extended_service(self): + expected_error_msg = "services with 'net: container' cannot be extended" - with mock.patch.object(config, 'load_yaml', return_value=other_config): - print(load_config()) - - with self.assertRaisesRegexp(ConfigurationError, 'net'): - other_config = {'web': {'net': 'container:db'}} - - with mock.patch.object(config, 'load_yaml', return_value=other_config): - print(load_config()) - - other_config = {'web': {'net': 'host'}} - - with mock.patch.object(config, 'load_yaml', return_value=other_config): - print(load_config()) + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + load_from_filename('tests/fixtures/extends/invalid-net.yml') def test_volume_path(self): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') From 4a8b2947caae7151db39a425e71f0f66dfd060ea Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 26 Aug 2015 13:23:29 +0100 Subject: [PATCH 114/337] Interpolate extended config This refactoring is now really coming together. Construction is happening in the __init__, which is a constructor and helps clean up the design and clarity of intent of the code. We can now see (nearly) everything that is being constructed when a ServiceLoader is created. It needs all of these data constructs to perform the domain logic and actions. Which are now clearer to see and moving more towards the principle of functions doing (mostly)one thing and function names being more descriptive. resolve_extends is now concerned with the resolving of extends, rather than the construction, validation, pre processing and *then* resolving of extends. Happy days :) Signed-off-by: Mazz Mosley --- compose/config/config.py | 44 ++++++++++--------- compose/config/validation.py | 8 ++++ .../extends/valid-interpolation-2.yml | 3 ++ .../fixtures/extends/valid-interpolation.yml | 5 +++ tests/unit/config_test.py | 11 +++++ 5 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/extends/valid-interpolation-2.yml create mode 100644 tests/fixtures/extends/valid-interpolation.yml diff --git a/compose/config/config.py b/compose/config/config.py index 65a5b5472d..e3ba2aeb83 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -11,6 +11,7 @@ from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables from .validation import validate_against_schema +from .validation import validate_extended_service_exists from .validation import validate_extends_file_path from .validation import validate_service_names from .validation import validate_top_level_object @@ -178,12 +179,25 @@ class ServiceLoader(object): self.service_dict['extends'], self.filename ) - self.extended_config_path = self.get_extended_config_path( self.service_dict['extends'] ) - extended_config = load_yaml(self.extended_config_path) - validate_against_schema(extended_config) + self.extended_service_name = self.service_dict['extends']['service'] + + full_extended_config = pre_process_config( + load_yaml(self.extended_config_path) + ) + + validate_extended_service_exists( + self.extended_service_name, + full_extended_config, + self.extended_config_path + ) + validate_against_schema(full_extended_config) + + self.extended_config = full_extended_config[self.extended_service_name] + else: + self.extended_config = None def detect_cycle(self, name): if self.signature(name) in self.already_seen: @@ -214,40 +228,28 @@ class ServiceLoader(object): self.service_dict['environment'] = env def resolve_extends(self): - if 'extends' not in self.service_dict: + if self.extended_config is None: return self.service_dict - extends_options = self.service_dict['extends'] service_name = self.service_dict['name'] - other_config_path = self.extended_config_path other_working_dir = os.path.dirname(self.extended_config_path) other_already_seen = self.already_seen + [self.signature(service_name)] - base_service = extends_options['service'] - other_config = load_yaml(other_config_path) - - if base_service not in other_config: - msg = ( - "Cannot extend service '%s' in %s: Service not found" - ) % (base_service, other_config_path) - raise ConfigurationError(msg) - - other_service_dict = other_config[base_service] other_loader = ServiceLoader( working_dir=other_working_dir, - filename=other_config_path, + filename=self.extended_config_path, service_name=service_name, - service_dict=other_service_dict, + service_dict=self.extended_config, already_seen=other_already_seen, ) - other_loader.detect_cycle(extends_options['service']) + other_loader.detect_cycle(self.extended_service_name) other_service_dict = other_loader.make_service_dict() validate_extended_service_dict( other_service_dict, - filename=other_config_path, - service=extends_options['service'], + filename=self.extended_config_path, + service=self.extended_service_name, ) return merge_service_dicts(other_service_dict, self.service_dict) diff --git a/compose/config/validation.py b/compose/config/validation.py index 1ae8981ca6..304e7e7600 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -79,6 +79,14 @@ def validate_extends_file_path(service_name, extends_options, filename): ) +def validate_extended_service_exists(extended_service_name, full_extended_config, extended_config_path): + if extended_service_name not in full_extended_config: + msg = ( + "Cannot extend service '%s' in %s: Service not found" + ) % (extended_service_name, extended_config_path) + raise ConfigurationError(msg) + + def get_unsupported_config_msg(service_name, error_key): msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) if error_key in DOCKER_CONFIG_HINTS: diff --git a/tests/fixtures/extends/valid-interpolation-2.yml b/tests/fixtures/extends/valid-interpolation-2.yml new file mode 100644 index 0000000000..cb7bd93fc2 --- /dev/null +++ b/tests/fixtures/extends/valid-interpolation-2.yml @@ -0,0 +1,3 @@ +web: + build: '.' + hostname: "host-${HOSTNAME_VALUE}" diff --git a/tests/fixtures/extends/valid-interpolation.yml b/tests/fixtures/extends/valid-interpolation.yml new file mode 100644 index 0000000000..68e8740fb4 --- /dev/null +++ b/tests/fixtures/extends/valid-interpolation.yml @@ -0,0 +1,5 @@ +myweb: + extends: + service: web + file: valid-interpolation-2.yml + command: top diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 98ae513854..7624bbdfd5 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -914,6 +914,17 @@ class ExtendsTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): load_from_filename('tests/fixtures/extends/invalid-net.yml') + @mock.patch.dict(os.environ) + def test_valid_interpolation_in_extended_service(self): + os.environ.update( + HOSTNAME_VALUE="penguin", + ) + expected_interpolated_value = "host-penguin" + + service_dicts = load_from_filename('tests/fixtures/extends/valid-interpolation.yml') + for service in service_dicts: + self.assertTrue(service['hostname'], expected_interpolated_value) + def test_volume_path(self): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') From 950577d60f7d9ff76c1087f0f93de97303975d71 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 28 Aug 2015 18:49:17 +0100 Subject: [PATCH 115/337] Split validation into fields and service We want to give feedback to the user as soon as possible about the validity of the config supplied for the services. When extending a service, we can validate that the fields are correct against our schema but we must wait until the *end* of the extends cycle once all of the extended dicts have been merged into the service dict, to perform the final validation check on the config to ensure it is a complete valid service. Doing this before that had happened resulted in false reports of invalid config, as common config when split out, by itself, is not a valid service but *is* valid config to be included. Signed-off-by: Mazz Mosley --- compose/config/config.py | 11 ++++-- .../{schema.json => fields_schema.json} | 18 --------- compose/config/service_schema.json | 39 +++++++++++++++++++ compose/config/validation.py | 18 +++++++-- .../extends/service-with-invalid-schema.yml | 3 +- .../service-with-valid-composite-extends.yml | 5 +++ .../fixtures/extends/valid-common-config.yml | 6 +++ tests/fixtures/extends/valid-common.yml | 3 ++ .../extends/valid-composite-extends.yml | 2 + tests/unit/config_test.py | 9 +++++ 10 files changed, 88 insertions(+), 26 deletions(-) rename compose/config/{schema.json => fields_schema.json} (90%) create mode 100644 compose/config/service_schema.json create mode 100644 tests/fixtures/extends/service-with-valid-composite-extends.yml create mode 100644 tests/fixtures/extends/valid-common-config.yml create mode 100644 tests/fixtures/extends/valid-common.yml create mode 100644 tests/fixtures/extends/valid-composite-extends.yml diff --git a/compose/config/config.py b/compose/config/config.py index e3ba2aeb83..736f5aeb39 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,7 +10,8 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables -from .validation import validate_against_schema +from .validation import validate_against_fields_schema +from .validation import validate_against_service_schema from .validation import validate_extended_service_exists from .validation import validate_extends_file_path from .validation import validate_service_names @@ -139,7 +140,7 @@ def load(config_details): config, working_dir, filename = config_details processed_config = pre_process_config(config) - validate_against_schema(processed_config) + validate_against_fields_schema(processed_config) service_dicts = [] @@ -193,7 +194,7 @@ class ServiceLoader(object): full_extended_config, self.extended_config_path ) - validate_against_schema(full_extended_config) + validate_against_fields_schema(full_extended_config) self.extended_config = full_extended_config[self.extended_service_name] else: @@ -205,6 +206,10 @@ class ServiceLoader(object): def make_service_dict(self): self.service_dict = self.resolve_extends() + + if not self.already_seen: + validate_against_service_schema(self.service_dict) + return process_container_options(self.service_dict, working_dir=self.working_dir) def resolve_environment(self): diff --git a/compose/config/schema.json b/compose/config/fields_schema.json similarity index 90% rename from compose/config/schema.json rename to compose/config/fields_schema.json index 94fe4fc522..92305c575a 100644 --- a/compose/config/schema.json +++ b/compose/config/fields_schema.json @@ -106,24 +106,6 @@ "working_dir": {"type": "string"} }, - "anyOf": [ - { - "required": ["build"], - "not": {"required": ["image"]} - }, - { - "required": ["image"], - "not": {"anyOf": [ - {"required": ["build"]}, - {"required": ["dockerfile"]} - ]} - }, - { - "required": ["extends"], - "not": {"required": ["build", "image"]} - } - ], - "dependencies": { "memswap_limit": ["mem_limit"] }, diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json new file mode 100644 index 0000000000..5cb5d6d070 --- /dev/null +++ b/compose/config/service_schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "type": "object", + + "properties": { + "name": {"type": "string"} + }, + + "required": ["name"], + + "allOf": [ + {"$ref": "fields_schema.json#/definitions/service"}, + {"$ref": "#/definitions/service_constraints"} + ], + + "definitions": { + "service_constraints": { + "anyOf": [ + { + "required": ["build"], + "not": {"required": ["image"]} + }, + { + "required": ["image"], + "not": {"anyOf": [ + {"required": ["build"]}, + {"required": ["dockerfile"]} + ]} + }, + { + "required": ["extends"], + "not": {"required": ["build", "image"]} + } + ] + } + } + +} diff --git a/compose/config/validation.py b/compose/config/validation.py index 304e7e7600..3ae5485a7d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,6 +5,7 @@ from functools import wraps from docker.utils.ports import split_port from jsonschema import Draft4Validator from jsonschema import FormatChecker +from jsonschema import RefResolver from jsonschema import ValidationError from .errors import ConfigurationError @@ -210,14 +211,25 @@ def process_errors(errors): return "\n".join(root_msgs + invalid_keys + required + type_errors + other_errors) -def validate_against_schema(config): +def validate_against_fields_schema(config): + schema_filename = "fields_schema.json" + return _validate_against_schema(config, schema_filename) + + +def validate_against_service_schema(config): + schema_filename = "service_schema.json" + return _validate_against_schema(config, schema_filename) + + +def _validate_against_schema(config, schema_filename): config_source_dir = os.path.dirname(os.path.abspath(__file__)) - schema_file = os.path.join(config_source_dir, "schema.json") + schema_file = os.path.join(config_source_dir, schema_filename) with open(schema_file, "r") as schema_fh: schema = json.load(schema_fh) - validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"])) + resolver = RefResolver('file://' + config_source_dir + '/', schema) + validation_output = Draft4Validator(schema, resolver=resolver, format_checker=FormatChecker(["ports"])) errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] if errors: diff --git a/tests/fixtures/extends/service-with-invalid-schema.yml b/tests/fixtures/extends/service-with-invalid-schema.yml index 90dc76a0ea..00c36647ef 100644 --- a/tests/fixtures/extends/service-with-invalid-schema.yml +++ b/tests/fixtures/extends/service-with-invalid-schema.yml @@ -1,5 +1,4 @@ myweb: extends: + file: valid-composite-extends.yml service: web -web: - command: top diff --git a/tests/fixtures/extends/service-with-valid-composite-extends.yml b/tests/fixtures/extends/service-with-valid-composite-extends.yml new file mode 100644 index 0000000000..6c419ed070 --- /dev/null +++ b/tests/fixtures/extends/service-with-valid-composite-extends.yml @@ -0,0 +1,5 @@ +myweb: + build: '.' + extends: + file: 'valid-composite-extends.yml' + service: web diff --git a/tests/fixtures/extends/valid-common-config.yml b/tests/fixtures/extends/valid-common-config.yml new file mode 100644 index 0000000000..d8f13e7a86 --- /dev/null +++ b/tests/fixtures/extends/valid-common-config.yml @@ -0,0 +1,6 @@ +myweb: + build: '.' + extends: + file: valid-common.yml + service: common-config + command: top diff --git a/tests/fixtures/extends/valid-common.yml b/tests/fixtures/extends/valid-common.yml new file mode 100644 index 0000000000..07ad68e3e7 --- /dev/null +++ b/tests/fixtures/extends/valid-common.yml @@ -0,0 +1,3 @@ +common-config: + environment: + - FOO=1 diff --git a/tests/fixtures/extends/valid-composite-extends.yml b/tests/fixtures/extends/valid-composite-extends.yml new file mode 100644 index 0000000000..8816c3f3b2 --- /dev/null +++ b/tests/fixtures/extends/valid-composite-extends.yml @@ -0,0 +1,2 @@ +web: + command: top diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 7624bbdfd5..8f4251cfa8 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -866,6 +866,7 @@ class ExtendsTest(unittest.TestCase): self.assertEquals(len(service), 1) self.assertIsInstance(service[0], dict) + self.assertEquals(service[0]['command'], "/bin/true") def test_extended_service_with_invalid_config(self): expected_error_msg = "Service 'myweb' has neither an image nor a build path specified" @@ -873,6 +874,10 @@ class ExtendsTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): load_from_filename('tests/fixtures/extends/service-with-invalid-schema.yml') + def test_extended_service_with_valid_config(self): + service = load_from_filename('tests/fixtures/extends/service-with-valid-composite-extends.yml') + self.assertEquals(service[0]['command'], "top") + def test_extends_file_defaults_to_self(self): """ Test not specifying a file in our extends options that the @@ -955,6 +960,10 @@ class ExtendsTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, err_msg): load_from_filename('tests/fixtures/extends/nonexistent-service.yml') + def test_partial_service_config_in_extends_is_still_valid(self): + dicts = load_from_filename('tests/fixtures/extends/valid-common-config.yml') + self.assertEqual(dicts[0]['environment'], {'FOO': '1'}) + class BuildPathTest(unittest.TestCase): def setUp(self): From 9fa6e42f5562be98a4541941f40327f248179b43 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 31 Aug 2015 17:52:00 +0100 Subject: [PATCH 116/337] process_errors handle both schemas Now the schema has been split into two, we need to modify the process_errors function to accomodate. Previously if an error.path was empty then it meant they were root errors. Now that service_schema checks after the service has been resolved, our service name is a key within the dictionary and so our root error logic check is no longer true. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 3ae5485a7d..59fb13948d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -125,7 +125,7 @@ def process_errors(errors): for error in errors: # handle root level errors - if len(error.path) == 0: + if len(error.path) == 0 and not error.instance.get('name'): if error.validator == 'type': msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." root_msgs.append(msg) @@ -137,11 +137,13 @@ def process_errors(errors): root_msgs.append(_clean_error_message(error.message)) else: - # handle service level errors - service_name = error.path[0] - - # pop the service name off our path - error.path.popleft() + try: + # field_schema errors will have service name on the path + service_name = error.path[0] + error.path.popleft() + except IndexError: + # service_schema errors will have the name in the instance + service_name = error.instance.get('name') if error.validator == 'additionalProperties': invalid_config_key = _parse_key_from_error_msg(error) From 4b487e3957bf6aad71358fb4fdc2c7bf952b1927 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 1 Sep 2015 16:14:02 +0100 Subject: [PATCH 117/337] Refactor extends back out of __init__ If make_service_dict is our factory function then we'll give it the responsibility of validation/construction and resolving. Signed-off-by: Mazz Mosley --- compose/config/config.py | 67 +++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 736f5aeb39..70eac267b1 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -170,42 +170,18 @@ class ServiceLoader(object): self.filename = filename self.already_seen = already_seen or [] self.service_dict = service_dict.copy() + self.service_name = service_name self.service_dict['name'] = service_name - self.resolve_environment() - - if 'extends' in self.service_dict: - validate_extends_file_path( - service_name, - self.service_dict['extends'], - self.filename - ) - self.extended_config_path = self.get_extended_config_path( - self.service_dict['extends'] - ) - self.extended_service_name = self.service_dict['extends']['service'] - - full_extended_config = pre_process_config( - load_yaml(self.extended_config_path) - ) - - validate_extended_service_exists( - self.extended_service_name, - full_extended_config, - self.extended_config_path - ) - validate_against_fields_schema(full_extended_config) - - self.extended_config = full_extended_config[self.extended_service_name] - else: - self.extended_config = None - def detect_cycle(self, name): if self.signature(name) in self.already_seen: raise CircularReference(self.already_seen + [self.signature(name)]) def make_service_dict(self): - self.service_dict = self.resolve_extends() + self.resolve_environment() + if 'extends' in self.service_dict: + self.validate_and_construct_extends() + self.service_dict = self.resolve_extends() if not self.already_seen: validate_against_service_schema(self.service_dict) @@ -232,19 +208,38 @@ class ServiceLoader(object): self.service_dict['environment'] = env + def validate_and_construct_extends(self): + validate_extends_file_path( + self.service_name, + self.service_dict['extends'], + self.filename + ) + self.extended_config_path = self.get_extended_config_path( + self.service_dict['extends'] + ) + self.extended_service_name = self.service_dict['extends']['service'] + + full_extended_config = pre_process_config( + load_yaml(self.extended_config_path) + ) + + validate_extended_service_exists( + self.extended_service_name, + full_extended_config, + self.extended_config_path + ) + validate_against_fields_schema(full_extended_config) + + self.extended_config = full_extended_config[self.extended_service_name] + def resolve_extends(self): - if self.extended_config is None: - return self.service_dict - - service_name = self.service_dict['name'] - other_working_dir = os.path.dirname(self.extended_config_path) - other_already_seen = self.already_seen + [self.signature(service_name)] + other_already_seen = self.already_seen + [self.signature(self.service_name)] other_loader = ServiceLoader( working_dir=other_working_dir, filename=self.extended_config_path, - service_name=service_name, + service_name=self.service_name, service_dict=self.extended_config, already_seen=other_already_seen, ) From 9979880c9fc5371f2e0a26fa4d43bfdad156263f Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 1 Sep 2015 17:00:52 +0100 Subject: [PATCH 118/337] Add in volume_driver I'd missed out this field by accident previously. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 92305c575a..299f6de4ff 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -102,6 +102,7 @@ "tty": {"type": "string"}, "user": {"type": "string"}, "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "working_dir": {"type": "string"} }, From f51a5431ec0c3318af5c39599805f20cd135d5f9 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 2 Sep 2015 14:58:45 +0100 Subject: [PATCH 119/337] Correct some schema field definitions Now validation is split in two, the integration tests helped highlight some places where the schema definition was incorrect. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 299f6de4ff..2a122b7a31 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -19,7 +19,12 @@ "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "command": {"$ref": "#/definitions/string_or_list"}, "container_name": {"type": "string"}, - "cpu_shares": {"type": "string"}, + "cpu_shares": { + "oneOf": [ + {"type": "number"}, + {"type": "string"} + ] + }, "cpuset": {"type": "string"}, "detach": {"type": "boolean"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, @@ -27,7 +32,7 @@ "dns_search": {"$ref": "#/definitions/string_or_list"}, "dockerfile": {"type": "string"}, "domainname": {"type": "string"}, - "entrypoint": {"type": "string"}, + "entrypoint": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": { @@ -75,7 +80,7 @@ }, "name": {"type": "string"}, "net": {"type": "string"}, - "pid": {"type": "string"}, + "pid": {"type": ["string", "null"]}, "ports": { "type": "array", @@ -94,10 +99,10 @@ "uniqueItems": true }, - "privileged": {"type": "string"}, + "privileged": {"type": "boolean"}, "read_only": {"type": "boolean"}, "restart": {"type": "string"}, - "security_opt": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "stdin_open": {"type": "string"}, "tty": {"type": "string"}, "user": {"type": "string"}, From 9b8e404d138a1999594231b12ba29c935b93eb69 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 2 Sep 2015 15:00:28 +0100 Subject: [PATCH 120/337] Pass service_name to process_errors Previously on Buffy... The process_errors was parsing a load of ValidationErrors that we get back from jsonschema which included assumptions about the state of the instance we're validating. Now it's split in two and we're doing field separate to service, those assumptions don't hold and we can't logically retrieve the service_name from the error parsing when we're doing service schema validation, have to explicitly pass this in. process_errors is high on my list for some future re-factoring to help make it a bit clearer, smaller state of doing things. Signed-off-by: Mazz Mosley --- compose/config/config.py | 2 +- compose/config/validation.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 70eac267b1..8df45b8a9f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -184,7 +184,7 @@ class ServiceLoader(object): self.service_dict = self.resolve_extends() if not self.already_seen: - validate_against_service_schema(self.service_dict) + validate_against_service_schema(self.service_dict, self.service_name) return process_container_options(self.service_dict, working_dir=self.working_dir) diff --git a/compose/config/validation.py b/compose/config/validation.py index 59fb13948d..632bdf03bd 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -95,7 +95,7 @@ def get_unsupported_config_msg(service_name, error_key): return msg -def process_errors(errors): +def process_errors(errors, service_name=None): """ jsonschema gives us an error tree full of information to explain what has gone wrong. Process each error and pull out relevant information and re-write @@ -137,13 +137,14 @@ def process_errors(errors): root_msgs.append(_clean_error_message(error.message)) else: - try: + if not service_name: # field_schema errors will have service name on the path service_name = error.path[0] error.path.popleft() - except IndexError: - # service_schema errors will have the name in the instance - service_name = error.instance.get('name') + else: + # service_schema errors have the service name passed in, as that + # is not available on error.path or necessarily error.instance + service_name = service_name if error.validator == 'additionalProperties': invalid_config_key = _parse_key_from_error_msg(error) @@ -218,12 +219,12 @@ def validate_against_fields_schema(config): return _validate_against_schema(config, schema_filename) -def validate_against_service_schema(config): +def validate_against_service_schema(config, service_name): schema_filename = "service_schema.json" - return _validate_against_schema(config, schema_filename) + return _validate_against_schema(config, schema_filename, service_name) -def _validate_against_schema(config, schema_filename): +def _validate_against_schema(config, schema_filename, service_name=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) schema_file = os.path.join(config_source_dir, schema_filename) @@ -235,5 +236,5 @@ def _validate_against_schema(config, schema_filename): errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] if errors: - error_msg = process_errors(errors) + error_msg = process_errors(errors, service_name) raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg)) From d31d24d19fc7faa54bd793e812ca3a4447afaa27 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 2 Sep 2015 15:03:29 +0100 Subject: [PATCH 121/337] Work around some coupling of links, net & volume_from This is minimal disruptive change I could make to ensure the service integration tests worked, now we have some validation happening. There is some coupling/entanglement/assumption going on here. Project when creating a service, using it's class method from_dicts performs some transformations on links, net & volume_from, which get passed on to Service when creating. Service itself, then performs some transformation on those values. This worked fine in the tests before because those options were merely passed on via make_service_dict. This is no longer true with our validation in place. You can't pass to ServiceLoader [(obj, 'string')] for links, the validation expects it to be a list of strings. Which it would be when passed into Project.from_dicts method. I think the tests need some re-factoring but for now, manually deleting keys out of the kwargs and then putting them back in for Service Creation allows the tests to continue. I am not super happy about this approach. Hopefully we can come back and improve it. Signed-off-by: Mazz Mosley --- tests/integration/testcases.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 58240d5ea4..4557c07b6a 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -31,11 +31,29 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] + links = kwargs.get('links', None) + volumes_from = kwargs.get('volumes_from', None) + net = kwargs.get('net', None) + + workaround_options = ['links', 'volumes_from', 'net'] + for key in workaround_options: + try: + del kwargs[key] + except KeyError: + pass + options = ServiceLoader(working_dir='.', filename=None, service_name=name, service_dict=kwargs).make_service_dict() labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() + if links: + options['links'] = links + if volumes_from: + options['volumes_from'] = volumes_from + if net: + options['net'] = net + return Service( project='composetest', client=self.client, From 6a399a5b2f1ed0e014fcb21bae80cae3b725e506 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 2 Sep 2015 15:41:25 +0100 Subject: [PATCH 122/337] Update tests to be compatible with validation Some were missing build '.' from their dicts, others were the incorrect type and one I've moved from integration to unit. Signed-off-by: Mazz Mosley --- tests/integration/service_test.py | 12 +----- tests/unit/config_test.py | 67 ++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0cf8cdb0ef..177471ffa9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -165,16 +165,6 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts)) - def test_create_container_with_extra_hosts_string(self): - extra_hosts = 'somehost:162.242.195.82' - service = self.create_service('db', extra_hosts=extra_hosts) - self.assertRaises(ConfigError, lambda: service.create_container()) - - def test_create_container_with_extra_hosts_list_of_dicts(self): - extra_hosts = [{'somehost': '162.242.195.82'}, {'otherhost': '50.31.209.229'}] - service = self.create_service('db', extra_hosts=extra_hosts) - self.assertRaises(ConfigError, lambda: service.create_container()) - def test_create_container_with_extra_hosts_dicts(self): extra_hosts = {'somehost': '162.242.195.82', 'otherhost': '50.31.209.229'} extra_hosts_list = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] @@ -515,7 +505,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(container['HostConfig']['Privileged'], True) def test_expose_does_not_publish_ports(self): - service = self.create_service('web', expose=[8000]) + service = self.create_service('web', expose=["8000"]) container = create_and_start_container(service).inspect() self.assertEqual(container['NetworkSettings']['Ports'], {'8000/tcp': None}) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 8f4251cfa8..21f1261ec0 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -206,6 +206,39 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_extra_hosts_string_raises_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'extra_hosts': 'somehost:162.242.195.82' + }}, + 'working_dir', + 'filename.yml' + ) + ) + + def test_config_extra_hosts_list_of_dicts_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'extra_hosts': [ + {'somehost': '162.242.195.82'}, + {'otherhost': '50.31.209.229'} + ] + }}, + 'working_dir', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) @@ -240,7 +273,7 @@ class InterpolationTest(unittest.TestCase): 'web': { 'image': '${FOO}', 'command': '${BAR}', - 'entrypoint': '${BAR}', + 'container_name': '${BAR}', }, }, working_dir='.', @@ -286,12 +319,13 @@ class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): os.environ['HOME'] = '/home/user' - d = make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') + d = make_service_dict('foo', {'build': '.', 'volumes': ['~:/container/path']}, working_dir='.') self.assertEqual(d['volumes'], ['/home/user:/container/path']) @mock.patch.dict(os.environ) def test_volume_binding_with_local_dir_name_raises_warning(self): def make_dict(**config): + config['build'] = '.' make_service_dict('foo', config, working_dir='.') with mock.patch('compose.config.config.log.warn') as warn: @@ -336,6 +370,7 @@ class InterpolationTest(unittest.TestCase): def test_named_volume_with_driver_does_not_expand(self): d = make_service_dict('foo', { + 'build': '.', 'volumes': ['namedvolume:/data'], 'volume_driver': 'foodriver', }, working_dir='.') @@ -345,6 +380,7 @@ class InterpolationTest(unittest.TestCase): def test_home_directory_with_driver_does_not_expand(self): os.environ['NAME'] = 'surprise!' d = make_service_dict('foo', { + 'build': '.', 'volumes': ['~:/data'], 'volume_driver': 'foodriver', }, working_dir='.') @@ -504,36 +540,36 @@ class MergeLabelsTest(unittest.TestCase): def test_no_override(self): service_dict = config.merge_service_dicts( - make_service_dict('foo', {'labels': ['foo=1', 'bar']}, 'tests/'), - make_service_dict('foo', {}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), + make_service_dict('foo', {'build': '.'}, 'tests/'), ) self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) def test_no_base(self): service_dict = config.merge_service_dicts( - make_service_dict('foo', {}, 'tests/'), - make_service_dict('foo', {'labels': ['foo=2']}, 'tests/'), + make_service_dict('foo', {'build': '.'}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=2']}, 'tests/'), ) self.assertEqual(service_dict['labels'], {'foo': '2'}) def test_override_explicit_value(self): service_dict = config.merge_service_dicts( - make_service_dict('foo', {'labels': ['foo=1', 'bar']}, 'tests/'), - make_service_dict('foo', {'labels': ['foo=2']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=2']}, 'tests/'), ) self.assertEqual(service_dict['labels'], {'foo': '2', 'bar': ''}) def test_add_explicit_value(self): service_dict = config.merge_service_dicts( - make_service_dict('foo', {'labels': ['foo=1', 'bar']}, 'tests/'), - make_service_dict('foo', {'labels': ['bar=2']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['bar=2']}, 'tests/'), ) self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': '2'}) def test_remove_explicit_value(self): service_dict = config.merge_service_dicts( - make_service_dict('foo', {'labels': ['foo=1', 'bar=2']}, 'tests/'), - make_service_dict('foo', {'labels': ['bar']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar=2']}, 'tests/'), + make_service_dict('foo', {'build': '.', 'labels': ['bar']}, 'tests/'), ) self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) @@ -615,6 +651,7 @@ class EnvTest(unittest.TestCase): service_dict = make_service_dict( 'foo', { + 'build': '.', 'environment': { 'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', @@ -633,7 +670,7 @@ class EnvTest(unittest.TestCase): def test_env_from_file(self): service_dict = make_service_dict( 'foo', - {'env_file': 'one.env'}, + {'build': '.', 'env_file': 'one.env'}, 'tests/fixtures/env', ) self.assertEqual( @@ -644,7 +681,7 @@ class EnvTest(unittest.TestCase): def test_env_from_multiple_files(self): service_dict = make_service_dict( 'foo', - {'env_file': ['one.env', 'two.env']}, + {'build': '.', 'env_file': ['one.env', 'two.env']}, 'tests/fixtures/env', ) self.assertEqual( @@ -666,7 +703,7 @@ class EnvTest(unittest.TestCase): os.environ['ENV_DEF'] = 'E3' service_dict = make_service_dict( 'foo', - {'env_file': 'resolve.env'}, + {'build': '.', 'env_file': 'resolve.env'}, 'tests/fixtures/env', ) self.assertEqual( From b54b932b54ea39054aeaab1273c8570001b90804 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 11:53:37 -0700 Subject: [PATCH 123/337] Exit gracefully when requests encounter a ReadTimeout exception. Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 2 +- compose/cli/main.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index ad67d5639f..91e4059c9c 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -34,5 +34,5 @@ def docker_client(): ca_cert=ca_cert, ) - timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) + timeout = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=timeout) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2ace13c22b..116c830021 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -10,6 +10,7 @@ from operator import attrgetter import dockerpty from docker.errors import APIError +from requests.exceptions import ReadTimeout from .. import __version__ from .. import legacy @@ -65,6 +66,12 @@ def main(): except NeedsBuildError as e: log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name) sys.exit(1) + except ReadTimeout as e: + log.error( + "HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n" + "If you encounter this issue regularly because of slow network conditions, consider setting " + "COMPOSE_HTTP_TIMEOUT to a higher value." + ) def setup_logging(): From 80c909299965243800401f061c17afc56a689cdd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 12:52:48 -0700 Subject: [PATCH 124/337] Document COMPOSE_HTTP_TIMEOUT env config Signed-off-by: Joffrey F --- docs/reference/overview.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 7425aa5e8a..52598737d6 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -44,6 +44,11 @@ the `docker` daemon. Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TLS verification. Defaults to `~/.docker`. +### COMPOSE\_HTTP\_TIMEOUT + +Configures the time (in seconds) a request to the Docker daemon is allowed to hang before Compose considers +it failed. Defaults to 60 seconds. + From 48466d7d824c17c321be6f4308166f34eff822f9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Sep 2015 16:08:18 -0400 Subject: [PATCH 125/337] Fix #1961 - docker-compose up should attach to all containers with no service names are specified, and add tests. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 5 ++++- tests/integration/cli_test.py | 28 +++++++++++++++++++++++----- tests/unit/cli/main_test.py | 10 ++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2ace13c22b..2d72646d10 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -583,8 +583,11 @@ class TopLevelCommand(Command): def build_log_printer(containers, service_names, monochrome): + if service_names: + containers = [c for c in containers if c.service in service_names] + return LogPrinter( - [c for c in containers if c.service in service_names], + containers, attach_params={"logs": True}, monochrome=monochrome) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 124ae55910..9606ef41f3 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -113,7 +113,7 @@ class CLITestCase(DockerClientTestCase): output = mock_stdout.getvalue() self.assertNotIn(cache_indicator, output) - def test_up(self): + def test_up_detached(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') another = self.project.get_service('another') @@ -121,10 +121,28 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(another.containers()), 1) # Ensure containers don't have stdin and stdout connected in -d mode - config = service.containers()[0].inspect()['Config'] - self.assertFalse(config['AttachStderr']) - self.assertFalse(config['AttachStdout']) - self.assertFalse(config['AttachStdin']) + container, = service.containers() + self.assertFalse(container.get('Config.AttachStderr')) + self.assertFalse(container.get('Config.AttachStdout')) + self.assertFalse(container.get('Config.AttachStdin')) + + def test_up_attached(self): + with mock.patch( + 'compose.cli.main.attach_to_logs', + autospec=True + ) as mock_attach: + self.command.dispatch(['up'], None) + _, args, kwargs = mock_attach.mock_calls[0] + _project, log_printer, _names, _timeout = args + + service = self.project.get_service('simple') + another = self.project.get_service('another') + self.assertEqual(len(service.containers()), 1) + self.assertEqual(len(another.containers()), 1) + self.assertEqual( + set(log_printer.containers), + set(self.project.containers()) + ) def test_up_with_links(self): self.command.base_dir = 'tests/fixtures/links-composefile' diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index 817e8f49b6..e3a4629e53 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -31,6 +31,16 @@ class CLIMainTestCase(unittest.TestCase): log_printer = build_log_printer(containers, service_names, True) self.assertEqual(log_printer.containers, containers[:3]) + def test_build_log_printer_all_services(self): + containers = [ + mock_container('web', 1), + mock_container('db', 1), + mock_container('other', 1), + ] + service_names = [] + log_printer = build_log_printer(containers, service_names, True) + self.assertEqual(log_printer.containers, containers) + def test_attach_to_logs(self): project = mock.create_autospec(Project) log_printer = mock.create_autospec(LogPrinter, containers=[]) From e634fe3fd6a768bcd5818dfb81b4c11cdc460853 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 14:28:57 -0700 Subject: [PATCH 126/337] Deprecation warning when DOCKER_CLIENT_TIMEOUT is used Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 91e4059c9c..e16549e87b 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -1,9 +1,12 @@ +import logging import os import ssl from docker import Client from docker import tls +log = logging.getLogger(__name__) + def docker_client(): """ @@ -34,5 +37,8 @@ def docker_client(): ca_cert=ca_cert, ) + if 'DOCKER_CLIENT_TIMEOUT' in os.environ: + log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') timeout = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) + return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=timeout) From b110bbe9e3f2c69e8f1dc8e990d16f4b016da955 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 14:31:30 -0700 Subject: [PATCH 127/337] Refined error message when timeout is encountered. Signed-off-by: Joffrey F --- compose/cli/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 116c830021..6618742996 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -2,6 +2,7 @@ from __future__ import print_function from __future__ import unicode_literals import logging +import os import re import signal import sys @@ -68,9 +69,9 @@ def main(): sys.exit(1) except ReadTimeout as e: log.error( - "HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n" + "An HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n" "If you encounter this issue regularly because of slow network conditions, consider setting " - "COMPOSE_HTTP_TIMEOUT to a higher value." + "COMPOSE_HTTP_TIMEOUT to a higher value (current value: %s)." % os.environ.get('COMPOSE_HTTP_TIMEOUT', 60) ) From f9c7346380dc9c3da7f465b8ac542673700db837 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 14:52:47 -0700 Subject: [PATCH 128/337] HTTP_TIMEOUT as importable constant for consistency Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 5 +++-- compose/cli/main.py | 4 ++-- compose/const.py | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index e16549e87b..601b0b9aab 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -5,6 +5,8 @@ import ssl from docker import Client from docker import tls +from ..const import HTTP_TIMEOUT + log = logging.getLogger(__name__) @@ -39,6 +41,5 @@ def docker_client(): if 'DOCKER_CLIENT_TIMEOUT' in os.environ: log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') - timeout = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) - return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=timeout) + return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=HTTP_TIMEOUT) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6618742996..13a8cef266 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -2,7 +2,6 @@ from __future__ import print_function from __future__ import unicode_literals import logging -import os import re import signal import sys @@ -17,6 +16,7 @@ from .. import __version__ from .. import legacy from ..config import parse_environment from ..const import DEFAULT_TIMEOUT +from ..const import HTTP_TIMEOUT from ..progress_stream import StreamOutputError from ..project import ConfigurationError from ..project import NoSuchService @@ -71,7 +71,7 @@ def main(): log.error( "An HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n" "If you encounter this issue regularly because of slow network conditions, consider setting " - "COMPOSE_HTTP_TIMEOUT to a higher value (current value: %s)." % os.environ.get('COMPOSE_HTTP_TIMEOUT', 60) + "COMPOSE_HTTP_TIMEOUT to a higher value (current value: %s)." % HTTP_TIMEOUT ) diff --git a/compose/const.py b/compose/const.py index 709c3a10d7..dbfa56b8cd 100644 --- a/compose/const.py +++ b/compose/const.py @@ -1,3 +1,4 @@ +import os DEFAULT_TIMEOUT = 10 LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' @@ -6,3 +7,4 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) From b165ae07c97e3af52528c829d136dd29c37da0a3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 1 Sep 2015 14:58:16 -0700 Subject: [PATCH 129/337] Configure PyInstaller using docker-compose.spec Signed-off-by: Aanand Prasad --- .dockerignore | 1 - .gitignore | 1 - docker-compose.spec | 24 ++++++++++++++++++++++++ script/build-linux-inner | 2 +- script/build-osx | 2 +- 5 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 docker-compose.spec diff --git a/.dockerignore b/.dockerignore index ba7e9155d5..5a4da301b1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,5 @@ build coverage-html dist -docker-compose.spec docs/_site venv diff --git a/.gitignore b/.gitignore index f6750c1ff5..1b0c50113f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,5 @@ /build /coverage-html /dist -/docker-compose.spec /docs/_site /venv diff --git a/docker-compose.spec b/docker-compose.spec new file mode 100644 index 0000000000..eae63914c8 --- /dev/null +++ b/docker-compose.spec @@ -0,0 +1,24 @@ +# -*- mode: python -*- + +block_cipher = None + +a = Analysis(['bin/docker-compose'], + pathex=['.'], + hiddenimports=[], + hookspath=None, + runtime_hooks=None, + cipher=block_cipher) + +pyz = PYZ(a.pure, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='docker-compose', + debug=False, + strip=None, + upx=True, + console=True ) diff --git a/script/build-linux-inner b/script/build-linux-inner index cfea838067..e5d290ebaa 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -8,6 +8,6 @@ mkdir -p `pwd`/dist chmod 777 `pwd`/dist pip install -r requirements-build.txt -su -c "pyinstaller -F bin/docker-compose" user +su -c "pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version diff --git a/script/build-osx b/script/build-osx index d99c1fb981..e1cc7038ac 100755 --- a/script/build-osx +++ b/script/build-osx @@ -8,6 +8,6 @@ virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install . -venv/bin/pyinstaller -F bin/docker-compose +venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version From 6a95f6d628933299ba0531b87a38c7f9a0c5dcc3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Sep 2015 18:00:08 -0700 Subject: [PATCH 130/337] custom timeout test rewrite Signed-off-by: Joffrey F --- tests/unit/cli/docker_client_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 5ccde73ad3..d497495b40 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -16,7 +16,7 @@ class DockerClientTestCase(unittest.TestCase): docker_client.docker_client() def test_docker_client_with_custom_timeout(self): - with mock.patch.dict(os.environ): - os.environ['DOCKER_CLIENT_TIMEOUT'] = timeout = "300" + timeout = 300 + with mock.patch('compose.cli.docker_client.HTTP_TIMEOUT', 300): client = docker_client.docker_client() - self.assertEqual(client.timeout, int(timeout)) + self.assertEqual(client.timeout, int(timeout)) From ecea79fd4e3ae4ee91c6e34c5230fec8739295f4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 1 Sep 2015 15:22:00 -0700 Subject: [PATCH 131/337] Bundle schema files Signed-off-by: Aanand Prasad --- docker-compose.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.spec b/docker-compose.spec index eae63914c8..678fc13238 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -17,6 +17,8 @@ exe = EXE(pyz, a.binaries, a.zipfiles, a.datas, + [('compose/config/fields_schema.json', 'compose/config/fields_schema.json', 'DATA')], + [('compose/config/service_schema.json', 'compose/config/service_schema.json', 'DATA')], name='docker-compose', debug=False, strip=None, From 7326608369d52656eae56202d3cc005300a17771 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 3 Sep 2015 11:44:08 +0100 Subject: [PATCH 132/337] expose array can contain either strings or numbers Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 6 +++++- tests/unit/config_test.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 2a122b7a31..f03ef7110c 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -42,7 +42,11 @@ ] }, - "expose": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "expose": { + "type": "array", + "items": {"type": ["string", "number"]}, + "uniqueItems": true + }, "extends": { "type": "object", diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 21f1261ec0..3f602fb593 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -239,6 +239,21 @@ class ConfigTest(unittest.TestCase): ) ) + def test_valid_config_which_allows_two_type_definitions(self): + expose_values = [["8000"], [8000]] + for expose in expose_values: + service = config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'expose': expose + }}, + 'working_dir', + 'filename.yml' + ) + ) + self.assertEqual(service[0]['expose'], expose) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From ef56523883a8c2d5bd2d48a556ece6a3b8b130f5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 18:56:40 -0400 Subject: [PATCH 133/337] Make external_links a regular service.option so that it's part of the config hash Signed-off-by: Daniel Nephin --- compose/service.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index b48f2e14bd..5942fca53d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -87,7 +87,16 @@ ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') class Service(object): - def __init__(self, name, client=None, project='default', links=None, external_links=None, volumes_from=None, net=None, **options): + def __init__( + self, + name, + client=None, + project='default', + links=None, + volumes_from=None, + net=None, + **options + ): if not re.match('^%s+$' % VALID_NAME_CHARS, project): raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS)) @@ -95,7 +104,6 @@ class Service(object): self.client = client self.project = project self.links = links or [] - self.external_links = external_links or [] self.volumes_from = volumes_from or [] self.net = net or None self.options = options @@ -528,7 +536,7 @@ class Service(object): links.append((container.name, self.name)) links.append((container.name, container.name)) links.append((container.name, container.name_without_project)) - for external_link in self.external_links: + for external_link in self.options.get('external_links') or []: if ':' not in external_link: link_name = external_link else: From e801981fed4049d0f7eab94ed969ec20f3ba8a76 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 19:21:22 -0400 Subject: [PATCH 134/337] Sort config keys Signed-off-by: Daniel Nephin --- compose/config/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8df45b8a9f..d9b06f3e7d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -22,9 +22,9 @@ from compose.cli.utils import find_candidates_in_parent_dirs DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', + 'command', 'cpu_shares', 'cpuset', - 'command', 'detach', 'devices', 'dns', @@ -38,12 +38,12 @@ DOCKER_CONFIG_KEYS = [ 'image', 'labels', 'links', + 'log_driver', + 'log_opt', 'mac_address', 'mem_limit', 'memswap_limit', 'net', - 'log_driver', - 'log_opt', 'pid', 'ports', 'privileged', From 08ba857807753f43a2b64844fe53ca70756bfa14 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 19:54:49 -0400 Subject: [PATCH 135/337] Cleanup some project logic. Signed-off-by: Daniel Nephin --- compose/project.py | 50 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/compose/project.py b/compose/project.py index 4e8696ba88..54d6c4434d 100644 --- a/compose/project.py +++ b/compose/project.py @@ -87,8 +87,14 @@ class Project(object): volumes_from = project.get_volumes_from(service_dict) net = project.get_net(service_dict) - project.services.append(Service(client=client, project=name, links=links, net=net, - volumes_from=volumes_from, **service_dict)) + project.services.append( + Service( + client=client, + project=name, + links=links, + net=net, + volumes_from=volumes_from, + **service_dict)) return project @property @@ -184,30 +190,26 @@ class Project(object): return volumes_from def get_net(self, service_dict): - if 'net' in service_dict: - net_name = get_service_name_from_net(service_dict.get('net')) + net = service_dict.pop('net', None) + if not net: + return - if net_name: - try: - net = self.get_service(net_name) - except NoSuchService: - try: - net = Container.from_id(self.client, net_name) - except APIError: - raise ConfigurationError( - 'Service "%s" is trying to use the network of "%s", ' - 'which is not the name of a service or container.' % ( - service_dict['name'], - net_name)) - else: - net = service_dict['net'] + net_name = get_service_name_from_net(net) + if not net_name: + return net - del service_dict['net'] - - else: - net = None - - return net + try: + return self.get_service(net_name) + except NoSuchService: + pass + try: + return Container.from_id(self.client, net_name) + except APIError: + raise ConfigurationError( + 'Service "%s" is trying to use the network of "%s", ' + 'which is not the name of a service or container.' % ( + service_dict['name'], + net_name)) def start(self, service_names=None, **options): for service in self.get_services(service_names): From c183e52502da8efd3e60f104b4d25f0577f55c04 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 1 Sep 2015 19:40:15 -0400 Subject: [PATCH 136/337] Fixes #1757 - include all service properties in the config_dict() Signed-off-by: Daniel Nephin --- compose/service.py | 3 +++ tests/integration/state_test.py | 22 +++++++++++++++++ tests/unit/service_test.py | 43 ++++++++++++++++++++++++++++++++- 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 5942fca53d..f60d57bfd8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -488,6 +488,9 @@ class Service(object): return { 'options': self.options, 'image_id': self.image()['Id'], + 'links': [(service.name, alias) for service, alias in self.links], + 'net': self.get_net_name() or getattr(self.net, 'id', self.net), + 'volumes_from': self.get_volumes_from_names(), } def get_dependency_names(self): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index b3dd42d996..d077f094d0 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -1,3 +1,7 @@ +""" +Integration tests which cover state convergence (aka smart recreate) performed +by `docker-compose up`. +""" from __future__ import unicode_literals import os @@ -151,6 +155,24 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.assertEqual(new_containers - old_containers, set()) + def test_service_removed_while_down(self): + next_cfg = { + 'web': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + }, + 'nginx': self.cfg['nginx'], + } + + containers = self.run_up(self.cfg) + self.assertEqual(len(containers), 3) + + project = self.make_project(self.cfg) + project.stop(timeout=1) + + containers = self.run_up(next_cfg) + self.assertEqual(len(containers), 2) + def converge(service, allow_recreate=True, diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index aa6d4d74f4..3981cad207 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -189,7 +189,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual( opts['labels'][LABEL_CONFIG_HASH], - 'b30306d0a73b67f67a45b99b88d36c359e470e6fa0c04dda1cf62d2087205b81') + '3c85881a8903b9d73a06c41860c8be08acce1494ab4cf8408375966dccd714de') self.assertEqual( opts['environment'], { @@ -331,6 +331,47 @@ class ServiceTest(unittest.TestCase): self.assertEqual(self.mock_client.build.call_count, 1) self.assertFalse(self.mock_client.build.call_args[1]['pull']) + def test_config_dict(self): + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + service = Service( + 'foo', + image='example.com/foo', + client=self.mock_client, + net=Service('other'), + links=[(Service('one'), 'one')], + volumes_from=[Service('two')]) + + config_dict = service.config_dict() + expected = { + 'image_id': 'abcd', + 'options': {'image': 'example.com/foo'}, + 'links': [('one', 'one')], + 'net': 'other', + 'volumes_from': ['two'], + } + self.assertEqual(config_dict, expected) + + def test_config_dict_with_net_from_container(self): + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + container = Container( + self.mock_client, + {'Id': 'aaabbb', 'Name': '/foo_1'}) + service = Service( + 'foo', + image='example.com/foo', + client=self.mock_client, + net=container) + + config_dict = service.config_dict() + expected = { + 'image_id': 'abcd', + 'options': {'image': 'example.com/foo'}, + 'links': [], + 'net': 'aaabbb', + 'volumes_from': [], + } + self.assertEqual(config_dict, expected) + def mock_get_image(images): if images: From 187ad4ce26401aaa10984c3c9a9782d6b2efdb87 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 3 Sep 2015 13:02:46 -0400 Subject: [PATCH 137/337] Refactor network_mode logic out of Service. Signed-off-by: Daniel Nephin --- compose/project.py | 11 +++-- compose/service.py | 88 +++++++++++++++++++++++++------------- tests/unit/project_test.py | 6 +-- tests/unit/service_test.py | 48 ++++++++++++++++++++- 4 files changed, 116 insertions(+), 37 deletions(-) diff --git a/compose/project.py b/compose/project.py index 54d6c4434d..8db20e7667 100644 --- a/compose/project.py +++ b/compose/project.py @@ -14,7 +14,10 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container from .legacy import check_for_legacy_containers +from .service import ContainerNet +from .service import Net from .service import Service +from .service import ServiceNet from .utils import parallel_execute @@ -192,18 +195,18 @@ class Project(object): def get_net(self, service_dict): net = service_dict.pop('net', None) if not net: - return + return Net(None) net_name = get_service_name_from_net(net) if not net_name: - return net + return Net(net) try: - return self.get_service(net_name) + return ServiceNet(self.get_service(net_name)) except NoSuchService: pass try: - return Container.from_id(self.client, net_name) + return ContainerNet(Container.from_id(self.client, net_name)) except APIError: raise ConfigurationError( 'Service "%s" is trying to use the network of "%s", ' diff --git a/compose/service.py b/compose/service.py index f60d57bfd8..bfc6f904e3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -105,7 +105,7 @@ class Service(object): self.project = project self.links = links or [] self.volumes_from = volumes_from or [] - self.net = net or None + self.net = net or Net(None) self.options = options def containers(self, stopped=False, one_off=False, filters={}): @@ -489,12 +489,12 @@ class Service(object): 'options': self.options, 'image_id': self.image()['Id'], 'links': [(service.name, alias) for service, alias in self.links], - 'net': self.get_net_name() or getattr(self.net, 'id', self.net), + 'net': self.net.id, 'volumes_from': self.get_volumes_from_names(), } def get_dependency_names(self): - net_name = self.get_net_name() + net_name = self.net.service_name return (self.get_linked_names() + self.get_volumes_from_names() + ([net_name] if net_name else [])) @@ -505,12 +505,6 @@ class Service(object): def get_volumes_from_names(self): return [s.name for s in self.volumes_from if isinstance(s, Service)] - def get_net_name(self): - if isinstance(self.net, Service): - return self.net.name - else: - return - def get_container_name(self, number, one_off=False): # TODO: Implement issue #652 here return build_container_name(self.project, self.name, number, one_off) @@ -562,25 +556,6 @@ class Service(object): return volumes_from - def _get_net(self): - if not self.net: - return None - - if isinstance(self.net, Service): - containers = self.net.containers() - if len(containers) > 0: - net = 'container:' + containers[0].id - else: - log.warning("Warning: Service %s is trying to use reuse the network stack " - "of another service that is not running." % (self.net.name)) - net = None - elif isinstance(self.net, Container): - net = 'container:' + self.net.id - else: - net = self.net - - return net - def _get_container_create_options( self, override_options, @@ -694,7 +669,7 @@ class Service(object): binds=options.get('binds'), volumes_from=self._get_volumes_from(), privileged=privileged, - network_mode=self._get_net(), + network_mode=self.net.mode, devices=devices, dns=dns, dns_search=dns_search, @@ -793,6 +768,61 @@ class Service(object): stream_output(output, sys.stdout) +class Net(object): + """A `standard` network mode (ex: host, bridge)""" + + service_name = None + + def __init__(self, net): + self.net = net + + @property + def id(self): + return self.net + + mode = id + + +class ContainerNet(object): + """A network mode that uses a containers network stack.""" + + service_name = None + + def __init__(self, container): + self.container = container + + @property + def id(self): + return self.container.id + + @property + def mode(self): + return 'container:' + self.container.id + + +class ServiceNet(object): + """A network mode that uses a service's network stack.""" + + def __init__(self, service): + self.service = service + + @property + def id(self): + return self.service.name + + service_name = id + + @property + def mode(self): + containers = self.service.containers() + if containers: + return 'container:' + containers[0].id + + log.warn("Warning: Service %s is trying to use reuse the network stack " + "of another service that is not running." % (self.id)) + return None + + # Names diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 37ebe5148d..ce74eb30b7 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -221,7 +221,7 @@ class ProjectTest(unittest.TestCase): } ], self.mock_client) service = project.get_service('test') - self.assertEqual(service._get_net(), None) + self.assertEqual(service.net.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) def test_use_net_from_container(self): @@ -236,7 +236,7 @@ class ProjectTest(unittest.TestCase): } ], self.mock_client) service = project.get_service('test') - self.assertEqual(service._get_net(), 'container:' + container_id) + self.assertEqual(service.net.mode, 'container:' + container_id) def test_use_net_from_service(self): container_name = 'test_aaa_1' @@ -261,7 +261,7 @@ class ProjectTest(unittest.TestCase): ], self.mock_client) service = project.get_service('test') - self.assertEqual(service._get_net(), 'container:' + container_name) + self.assertEqual(service.net.mode, 'container:' + container_name) def test_container_without_name(self): self.mock_client.containers.return_value = [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 3981cad207..de973339b2 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -13,13 +13,16 @@ from compose.const import LABEL_SERVICE from compose.container import Container from compose.service import build_volume_binding from compose.service import ConfigError +from compose.service import ContainerNet from compose.service import get_container_data_volumes from compose.service import merge_volume_bindings from compose.service import NeedsBuildError +from compose.service import Net from compose.service import NoSuchImageError from compose.service import parse_repository_tag from compose.service import parse_volume_spec from compose.service import Service +from compose.service import ServiceNet class ServiceTest(unittest.TestCase): @@ -337,7 +340,7 @@ class ServiceTest(unittest.TestCase): 'foo', image='example.com/foo', client=self.mock_client, - net=Service('other'), + net=ServiceNet(Service('other')), links=[(Service('one'), 'one')], volumes_from=[Service('two')]) @@ -373,6 +376,49 @@ class ServiceTest(unittest.TestCase): self.assertEqual(config_dict, expected) +class NetTestCase(unittest.TestCase): + + def test_net(self): + net = Net('host') + self.assertEqual(net.id, 'host') + self.assertEqual(net.mode, 'host') + self.assertEqual(net.service_name, None) + + def test_net_container(self): + container_id = 'abcd' + net = ContainerNet(Container(None, {'Id': container_id})) + self.assertEqual(net.id, container_id) + self.assertEqual(net.mode, 'container:' + container_id) + self.assertEqual(net.service_name, None) + + def test_net_service(self): + container_id = 'bbbb' + service_name = 'web' + mock_client = mock.create_autospec(docker.Client) + mock_client.containers.return_value = [ + {'Id': container_id, 'Name': container_id, 'Image': 'abcd'}, + ] + + service = Service(name=service_name, client=mock_client) + net = ServiceNet(service) + + self.assertEqual(net.id, service_name) + self.assertEqual(net.mode, 'container:' + container_id) + self.assertEqual(net.service_name, service_name) + + def test_net_service_no_containers(self): + service_name = 'web' + mock_client = mock.create_autospec(docker.Client) + mock_client.containers.return_value = [] + + service = Service(name=service_name, client=mock_client) + net = ServiceNet(service) + + self.assertEqual(net.id, service_name) + self.assertEqual(net.mode, None) + self.assertEqual(net.service_name, service_name) + + def mock_get_image(images): if images: return images[0] From db9f577ad6cdadfb8eaa33b492fd513821ed57b6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 3 Sep 2015 13:13:22 -0400 Subject: [PATCH 138/337] Extract link names into a function. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- compose/service.py | 13 ++++++++----- tests/integration/project_test.py | 4 ++-- tests/integration/service_test.py | 7 ++++--- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2d72646d10..11d2d104c7 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -326,7 +326,7 @@ class TopLevelCommand(Command): log.warn(INSECURE_SSL_WARNING) if not options['--no-deps']: - deps = service.get_linked_names() + deps = service.get_linked_service_names() if len(deps) > 0: project.up( diff --git a/compose/service.py b/compose/service.py index bfc6f904e3..8dc1efa1d2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -488,19 +488,22 @@ class Service(object): return { 'options': self.options, 'image_id': self.image()['Id'], - 'links': [(service.name, alias) for service, alias in self.links], + 'links': self.get_link_names(), 'net': self.net.id, 'volumes_from': self.get_volumes_from_names(), } def get_dependency_names(self): net_name = self.net.service_name - return (self.get_linked_names() + + return (self.get_linked_service_names() + self.get_volumes_from_names() + ([net_name] if net_name else [])) - def get_linked_names(self): - return [s.name for (s, _) in self.links] + def get_linked_service_names(self): + return [service.name for (service, _) in self.links] + + def get_link_names(self): + return [(service.name, alias) for service, alias in self.links] def get_volumes_from_names(self): return [s.name for s in self.volumes_from if isinstance(s, Service)] @@ -784,7 +787,7 @@ class Net(object): class ContainerNet(object): - """A network mode that uses a containers network stack.""" + """A network mode that uses a container's network stack.""" service_name = None diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 51619cb5ec..fe63838fcb 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -112,7 +112,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') net = project.get_service('net') - self.assertEqual(web._get_net(), 'container:' + net.containers()[0].id) + self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) def test_net_from_container(self): net_container = Container.create( @@ -138,7 +138,7 @@ class ProjectTest(DockerClientTestCase): project.up() web = project.get_service('web') - self.assertEqual(web._get_net(), 'container:' + net_container.id) + self.assertEqual(web.net.mode, 'container:' + net_container.id) def test_start_pause_unpause_stop_kill_remove(self): web = self.create_service('web') diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 177471ffa9..b6257821dc 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -22,6 +22,7 @@ from compose.container import Container from compose.service import build_extra_hosts from compose.service import ConfigError from compose.service import ConvergencePlan +from compose.service import Net from compose.service import Service @@ -707,17 +708,17 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(list(container.inspect()['HostConfig']['PortBindings'].keys()), ['8000/tcp']) def test_network_mode_none(self): - service = self.create_service('web', net='none') + service = self.create_service('web', net=Net('none')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'none') def test_network_mode_bridged(self): - service = self.create_service('web', net='bridge') + service = self.create_service('web', net=Net('bridge')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'bridge') def test_network_mode_host(self): - service = self.create_service('web', net='host') + service = self.create_service('web', net=Net('host')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'host') From 9d7ad796a38ee79f7dd2c1436cadb6d2bb17b24e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 3 Sep 2015 14:11:44 -0400 Subject: [PATCH 139/337] bump requests to 2.7 to fix the ResponseNotReady() error, and add a missing default for tox posargs Signed-off-by: Daniel Nephin --- requirements.txt | 2 +- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index e93db7b361..587c04c5a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ docker-py==1.3.1 dockerpty==0.3.4 docopt==0.6.1 jsonschema==2.5.1 -requests==2.6.1 +requests==2.7.0 six==1.7.3 texttable==0.8.2 websocket-client==0.32.0 diff --git a/setup.py b/setup.py index e93dafc628..737e074c81 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def find_version(*file_paths): install_requires = [ 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, < 2.7', + 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', 'docker-py >= 1.3.1, < 1.4', diff --git a/tox.ini b/tox.ini index 71ab4fc9c9..4cb933dd71 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ commands = --cov-report html \ --cov-report term \ --cov-config=tox.ini \ - {posargs} + {posargs:tests} [testenv:pre-commit] skip_install = True From a1ec26435cbe41ad63e765bfd973ccf306a4e54c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 3 Sep 2015 18:30:50 -0700 Subject: [PATCH 140/337] Test against Docker 1.8.2 RC1 Signed-off-by: Aanand Prasad --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1d13c2b603..ba508742de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,13 +66,13 @@ RUN set -ex; \ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 -ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.1 +ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.2-rc1 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.7.1 -o /usr/local/bin/docker-1.7.1; \ chmod +x /usr/local/bin/docker-1.7.1; \ - curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.1 -o /usr/local/bin/docker-1.8.1; \ - chmod +x /usr/local/bin/docker-1.8.1 + curl https://test.docker.com/builds/Linux/x86_64/docker-1.8.2-rc1 -o /usr/local/bin/docker-1.8.2-rc1; \ + chmod +x /usr/local/bin/docker-1.8.2-rc1 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker From 000bf1c16a1e2181d4b6b2a580692ed3f48231c0 Mon Sep 17 00:00:00 2001 From: Charles Chan Date: Wed, 2 Sep 2015 21:06:25 -0700 Subject: [PATCH 141/337] Fix #1958. Remove release notes for old version of Docker Compose. Replace by link to the latest CHANGELOG in GitHub. Signed-off-by: Charles Chan --- docs/index.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4342b3686d..59bf200941 100644 --- a/docs/index.md +++ b/docs/index.md @@ -206,17 +206,8 @@ At this point, you have seen the basics of how Compose works. ## Release Notes -### Version 1.2.0 (April 7, 2015) - -For complete information on this release, see the [1.2.0 Milestone project page](https://github.com/docker/compose/wiki/1.2.0-Milestone-Project-Page). -In addition to bug fixes and refinements, this release adds the following: - -* The `extends` keyword, which adds the ability to extend services by sharing common configurations. For details, see -[PR #1088](https://github.com/docker/compose/pull/1088). - -* Better integration with Swarm. Swarm will now schedule inter-dependent -containers on the same host. For details, see -[PR #972](https://github.com/docker/compose/pull/972). +To see a detailed list of changes for past and current releases of Docker +Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). ## Getting help From 0484e22a84cf430871b32d1136d94d3083214f61 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Sep 2015 11:07:59 -0400 Subject: [PATCH 142/337] Add enum34 and use it to create a ConvergenceStrategy enum. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 31 ++++++++++++++--------- compose/project.py | 38 ++++++++-------------------- compose/service.py | 30 +++++++++++++++------- docs/index.md | 2 +- requirements.txt | 1 + setup.py | 3 ++- tests/integration/cli_test.py | 6 ++--- tests/integration/project_test.py | 7 ++--- tests/integration/resilience_test.py | 7 ++--- tests/integration/state_test.py | 25 ++++++------------ tests/unit/cli/main_test.py | 32 +++++++++++++++++++++++ 11 files changed, 105 insertions(+), 77 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 11d2d104c7..cf971844bc 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -19,6 +19,7 @@ from ..progress_stream import StreamOutputError from ..project import ConfigurationError from ..project import NoSuchService from ..service import BuildError +from ..service import ConvergenceStrategy from ..service import NeedsBuildError from .command import Command from .docopt_command import NoSuchCommand @@ -332,7 +333,7 @@ class TopLevelCommand(Command): project.up( service_names=deps, start_deps=True, - allow_recreate=False, + strategy=ConvergenceStrategy.never, ) tty = True @@ -515,29 +516,20 @@ class TopLevelCommand(Command): if options['--allow-insecure-ssl']: log.warn(INSECURE_SSL_WARNING) - detached = options['-d'] - monochrome = options['--no-color'] - start_deps = not options['--no-deps'] - allow_recreate = not options['--no-recreate'] - force_recreate = options['--force-recreate'] service_names = options['SERVICE'] timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) - if force_recreate and not allow_recreate: - raise UserError("--force-recreate and --no-recreate cannot be combined.") - to_attach = project.up( service_names=service_names, start_deps=start_deps, - allow_recreate=allow_recreate, - force_recreate=force_recreate, + strategy=convergence_strategy_from_opts(options), do_build=not options['--no-build'], timeout=timeout ) - if not detached: + if not options['-d']: log_printer = build_log_printer(to_attach, service_names, monochrome) attach_to_logs(project, log_printer, service_names, timeout) @@ -582,6 +574,21 @@ class TopLevelCommand(Command): print(get_version_info('full')) +def convergence_strategy_from_opts(options): + no_recreate = options['--no-recreate'] + force_recreate = options['--force-recreate'] + if force_recreate and no_recreate: + raise UserError("--force-recreate and --no-recreate cannot be combined.") + + if force_recreate: + return ConvergenceStrategy.always + + if no_recreate: + return ConvergenceStrategy.never + + return ConvergenceStrategy.changed + + def build_log_printer(containers, service_names, monochrome): if service_names: containers = [c for c in containers if c.service in service_names] diff --git a/compose/project.py b/compose/project.py index 8db20e7667..9a6e98e01d 100644 --- a/compose/project.py +++ b/compose/project.py @@ -15,6 +15,7 @@ from .const import LABEL_SERVICE from .container import Container from .legacy import check_for_legacy_containers from .service import ContainerNet +from .service import ConvergenceStrategy from .service import Net from .service import Service from .service import ServiceNet @@ -266,24 +267,16 @@ class Project(object): def up(self, service_names=None, start_deps=True, - allow_recreate=True, - force_recreate=False, + strategy=ConvergenceStrategy.changed, do_build=True, timeout=DEFAULT_TIMEOUT): - if force_recreate and not allow_recreate: - raise ValueError("force_recreate and allow_recreate are in conflict") - services = self.get_services(service_names, include_deps=start_deps) for service in services: service.remove_duplicate_containers() - plans = self._get_convergence_plans( - services, - allow_recreate=allow_recreate, - force_recreate=force_recreate, - ) + plans = self._get_convergence_plans(services, strategy) return [ container @@ -295,11 +288,7 @@ class Project(object): ) ] - def _get_convergence_plans(self, - services, - allow_recreate=True, - force_recreate=False): - + def _get_convergence_plans(self, services, strategy): plans = {} for service in services: @@ -310,20 +299,13 @@ class Project(object): and plans[name].action == 'recreate' ] - if updated_dependencies and allow_recreate: - log.debug( - '%s has upstream changes (%s)', - service.name, ", ".join(updated_dependencies), - ) - plan = service.convergence_plan( - allow_recreate=allow_recreate, - force_recreate=True, - ) + if updated_dependencies and strategy.allows_recreate: + log.debug('%s has upstream changes (%s)', + service.name, + ", ".join(updated_dependencies)) + plan = service.convergence_plan(ConvergenceStrategy.always) else: - plan = service.convergence_plan( - allow_recreate=allow_recreate, - force_recreate=force_recreate, - ) + plan = service.convergence_plan(strategy) plans[service.name] = plan diff --git a/compose/service.py b/compose/service.py index 8dc1efa1d2..be74ca3a2e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -8,6 +8,7 @@ import sys from collections import namedtuple from operator import attrgetter +import enum import six from docker.errors import APIError from docker.utils import create_host_config @@ -86,6 +87,20 @@ ServiceName = namedtuple('ServiceName', 'project service number') ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') +@enum.unique +class ConvergenceStrategy(enum.Enum): + """Enumeration for all possible convergence strategies. Values refer to + when containers should be recreated. + """ + changed = 1 + always = 2 + never = 3 + + @property + def allows_recreate(self): + return self is not type(self).never + + class Service(object): def __init__( self, @@ -326,22 +341,19 @@ class Service(object): else: return self.options['image'] - def convergence_plan(self, - allow_recreate=True, - force_recreate=False): - - if force_recreate and not allow_recreate: - raise ValueError("force_recreate and allow_recreate are in conflict") - + def convergence_plan(self, strategy=ConvergenceStrategy.changed): containers = self.containers(stopped=True) if not containers: return ConvergencePlan('create', []) - if not allow_recreate: + if strategy is ConvergenceStrategy.never: return ConvergencePlan('start', containers) - if force_recreate or self._containers_have_diverged(containers): + if ( + strategy is ConvergenceStrategy.always or + self._containers_have_diverged(containers) + ): return ConvergencePlan('recreate', containers) stopped = [c for c in containers if not c.is_running] diff --git a/docs/index.md b/docs/index.md index 59bf200941..4e4f58dafd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -206,7 +206,7 @@ At this point, you have seen the basics of how Compose works. ## Release Notes -To see a detailed list of changes for past and current releases of Docker +To see a detailed list of changes for past and current releases of Docker Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). ## Getting help diff --git a/requirements.txt b/requirements.txt index 587c04c5a4..666efcd268 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ PyYAML==3.10 docker-py==1.3.1 dockerpty==0.3.4 docopt==0.6.1 +enum34==1.0.4 jsonschema==2.5.1 requests==2.7.0 six==1.7.3 diff --git a/setup.py b/setup.py index 737e074c81..29c5299e9d 100644 --- a/setup.py +++ b/setup.py @@ -45,8 +45,9 @@ tests_require = [ ] -if sys.version_info[:1] < (3,): +if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') + install_requires.append('enum34 >= 1.0.4, < 2') setup( diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 9606ef41f3..4a80d33695 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -223,7 +223,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(config['AttachStdin']) @mock.patch('dockerpty.start') - def test_run_service_with_links(self, __): + def test_run_service_with_links(self, _): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', 'web', '/bin/true'], None) db = self.project.get_service('db') @@ -232,14 +232,14 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(console.containers()), 0) @mock.patch('dockerpty.start') - def test_run_with_no_deps(self, __): + def test_run_with_no_deps(self, _): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', '--no-deps', 'web', '/bin/true'], None) db = self.project.get_service('db') self.assertEqual(len(db.containers()), 0) @mock.patch('dockerpty.start') - def test_run_does_not_recreate_linked_containers(self, __): + def test_run_does_not_recreate_linked_containers(self, _): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d', 'db'], None) db = self.project.get_service('db') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index fe63838fcb..ad49ad10a8 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -5,6 +5,7 @@ from compose import config from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project +from compose.service import ConvergenceStrategy def build_service_dicts(service_config): @@ -224,7 +225,7 @@ class ProjectTest(DockerClientTestCase): old_db_id = project.containers()[0].id db_volume_path = project.containers()[0].get('Volumes./etc') - project.up(force_recreate=True) + project.up(strategy=ConvergenceStrategy.always) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] @@ -243,7 +244,7 @@ class ProjectTest(DockerClientTestCase): old_db_id = project.containers()[0].id db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] - project.up(allow_recreate=False) + project.up(strategy=ConvergenceStrategy.never) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] @@ -267,7 +268,7 @@ class ProjectTest(DockerClientTestCase): old_db_id = old_containers[0].id db_volume_path = old_containers[0].inspect()['Volumes']['/var/db'] - project.up(allow_recreate=False) + project.up(strategy=ConvergenceStrategy.never) new_containers = project.containers(stopped=True) self.assertEqual(len(new_containers), 2) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 82a4680d8b..befd72c7f8 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from .. import mock from .testcases import DockerClientTestCase from compose.project import Project +from compose.service import ConvergenceStrategy class ResilienceTest(DockerClientTestCase): @@ -16,14 +17,14 @@ class ResilienceTest(DockerClientTestCase): self.host_path = container.get('Volumes')['/var/db'] def test_successful_recreate(self): - self.project.up(force_recreate=True) + self.project.up(strategy=ConvergenceStrategy.always) container = self.db.containers()[0] self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) def test_create_failure(self): with mock.patch('compose.service.Service.create_container', crash): with self.assertRaises(Crash): - self.project.up(force_recreate=True) + self.project.up(strategy=ConvergenceStrategy.always) self.project.up() container = self.db.containers()[0] @@ -32,7 +33,7 @@ class ResilienceTest(DockerClientTestCase): def test_start_failure(self): with mock.patch('compose.service.Service.start_container', crash): with self.assertRaises(Crash): - self.project.up(force_recreate=True) + self.project.up(strategy=ConvergenceStrategy.always) self.project.up() container = self.db.containers()[0] diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index d077f094d0..93d0572a08 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -12,6 +12,7 @@ from .testcases import DockerClientTestCase from compose import config from compose.const import LABEL_CONFIG_HASH from compose.project import Project +from compose.service import ConvergenceStrategy class ProjectTestCase(DockerClientTestCase): @@ -151,7 +152,9 @@ class ProjectWithDependenciesTest(ProjectTestCase): old_containers = self.run_up(self.cfg) self.cfg['db']['environment'] = {'NEW_VAR': '1'} - new_containers = self.run_up(self.cfg, allow_recreate=False) + new_containers = self.run_up( + self.cfg, + strategy=ConvergenceStrategy.never) self.assertEqual(new_containers - old_containers, set()) @@ -175,23 +178,11 @@ class ProjectWithDependenciesTest(ProjectTestCase): def converge(service, - allow_recreate=True, - force_recreate=False, + strategy=ConvergenceStrategy.changed, do_build=True): - """ - If a container for this service doesn't exist, create and start one. If there are - any, stop them, create+start new ones, and remove the old containers. - """ - plan = service.convergence_plan( - allow_recreate=allow_recreate, - force_recreate=force_recreate, - ) - - return service.execute_convergence_plan( - plan, - do_build=do_build, - timeout=1, - ) + """Create a converge plan from a strategy and execute the plan.""" + plan = service.convergence_plan(strategy) + return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index e3a4629e53..a5b369808b 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -1,10 +1,13 @@ from __future__ import absolute_import from compose import container +from compose.cli.errors import UserError from compose.cli.log_printer import LogPrinter from compose.cli.main import attach_to_logs from compose.cli.main import build_log_printer +from compose.cli.main import convergence_strategy_from_opts from compose.project import Project +from compose.service import ConvergenceStrategy from tests import mock from tests import unittest @@ -55,3 +58,32 @@ class CLIMainTestCase(unittest.TestCase): project.stop.assert_called_once_with( service_names=service_names, timeout=timeout) + + +class ConvergeStrategyFromOptsTestCase(unittest.TestCase): + + def test_invalid_opts(self): + options = {'--force-recreate': True, '--no-recreate': True} + with self.assertRaises(UserError): + convergence_strategy_from_opts(options) + + def test_always(self): + options = {'--force-recreate': True, '--no-recreate': False} + self.assertEqual( + convergence_strategy_from_opts(options), + ConvergenceStrategy.always + ) + + def test_never(self): + options = {'--force-recreate': False, '--no-recreate': True} + self.assertEqual( + convergence_strategy_from_opts(options), + ConvergenceStrategy.never + ) + + def test_changed(self): + options = {'--force-recreate': False, '--no-recreate': False} + self.assertEqual( + convergence_strategy_from_opts(options), + ConvergenceStrategy.changed + ) From 0f60c783fa4feba0e4a1f33ac662e8b046355fe5 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 4 Sep 2015 17:01:44 +0100 Subject: [PATCH 143/337] Remove trailing white space Signed-off-by: Mazz Mosley --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 59bf200941..4e4f58dafd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -206,7 +206,7 @@ At this point, you have seen the basics of how Compose works. ## Release Notes -To see a detailed list of changes for past and current releases of Docker +To see a detailed list of changes for past and current releases of Docker Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). ## Getting help From 31e8137452dfefbc3fc36c754d9839e89978542d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 4 Sep 2015 17:20:21 +0100 Subject: [PATCH 144/337] Running a single test command updated Signed-off-by: Mazz Mosley --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a94aa9904b..62bf415c7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,8 +64,8 @@ you can specify a test directory, file, module, class or method: $ script/test tests/unit $ script/test tests/unit/cli_test.py - $ script/test tests.integration.service_test - $ script/test tests.integration.service_test:ServiceTest.test_containers + $ script/test tests/unit/config_test.py::ConfigTest + $ script/test tests/unit/config_test.py::ConfigTest::test_load ## Finding things to work on From 6da7a9194c8d02dce71b9b70c499c3abb64ede5f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 4 Sep 2015 17:43:12 -0700 Subject: [PATCH 145/337] Remove or space out suspension dots after service name for easier copy-pasting Signed-off-by: Joffrey F --- compose/service.py | 20 ++++++++++---------- compose/utils.py | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8dc1efa1d2..1a34f50c4f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -143,27 +143,27 @@ class Service(object): # TODO: remove these functions, project takes care of starting/stopping, def stop(self, **options): for c in self.containers(): - log.info("Stopping %s..." % c.name) + log.info("Stopping %s" % c.name) c.stop(**options) def pause(self, **options): for c in self.containers(filters={'status': 'running'}): - log.info("Pausing %s..." % c.name) + log.info("Pausing %s" % c.name) c.pause(**options) def unpause(self, **options): for c in self.containers(filters={'status': 'paused'}): - log.info("Unpausing %s..." % c.name) + log.info("Unpausing %s" % c.name) c.unpause() def kill(self, **options): for c in self.containers(): - log.info("Killing %s..." % c.name) + log.info("Killing %s" % c.name) c.kill(**options) def restart(self, **options): for c in self.containers(): - log.info("Restarting %s..." % c.name) + log.info("Restarting %s" % c.name) c.restart(**options) # end TODO @@ -289,7 +289,7 @@ class Service(object): ) if 'name' in container_options and not quiet: - log.info("Creating %s..." % container_options['name']) + log.info("Creating %s" % container_options['name']) return Container.create(self.client, **container_options) @@ -423,7 +423,7 @@ class Service(object): volumes can be copied to the new container, before the original container is removed. """ - log.info("Recreating %s..." % container.name) + log.info("Recreating %s" % container.name) try: container.stop(timeout=timeout) except APIError as e: @@ -453,7 +453,7 @@ class Service(object): if container.is_running: return container else: - log.info("Starting %s..." % container.name) + log.info("Starting %s" % container.name) return self.start_container(container) def start_container(self, container): @@ -462,7 +462,7 @@ class Service(object): def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): for c in self.duplicate_containers(): - log.info('Removing %s...' % c.name) + log.info('Removing %s' % c.name) c.stop(timeout=timeout) c.remove() @@ -689,7 +689,7 @@ class Service(object): ) def build(self, no_cache=False): - log.info('Building %s...' % self.name) + log.info('Building %s' % self.name) path = self.options['build'] # python2 os.path() doesn't support unicode, so we need to encode it to diff --git a/compose/utils.py b/compose/utils.py index 30284f97bd..690c5ffd53 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -90,13 +90,13 @@ def write_out_msg(stream, lines, msg_index, msg, status="done"): stream.write("%c[%dA" % (27, diff)) # erase stream.write("%c[2K\r" % 27) - stream.write("{} {}... {}\n".format(msg, obj_index, status)) + stream.write("{} {} ... {}\n".format(msg, obj_index, status)) # move back down stream.write("%c[%dB" % (27, diff)) else: diff = 0 lines.append(obj_index) - stream.write("{} {}... \r\n".format(msg, obj_index)) + stream.write("{} {} ... \r\n".format(msg, obj_index)) stream.flush() From 2468235472eb0a849dcad7a7838488cc9df6f8dc Mon Sep 17 00:00:00 2001 From: Lachlan Pease Date: Sun, 6 Sep 2015 11:55:36 +1000 Subject: [PATCH 146/337] Added support for IPC namespaces, fixes GH-1689 Signed-off-by: Lachlan Pease --- compose/config/config.py | 1 + compose/service.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8df45b8a9f..c23a541eea 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -36,6 +36,7 @@ DOCKER_CONFIG_KEYS = [ 'extra_hosts', 'hostname', 'image', + 'ipc', 'labels', 'links', 'mac_address', diff --git a/compose/service.py b/compose/service.py index b48f2e14bd..bf65888c78 100644 --- a/compose/service.py +++ b/compose/service.py @@ -44,6 +44,7 @@ DOCKER_START_KEYS = [ 'dns_search', 'env_file', 'extra_hosts', + 'ipc', 'read_only', 'net', 'log_driver', @@ -696,7 +697,8 @@ class Service(object): extra_hosts=extra_hosts, read_only=read_only, pid_mode=pid, - security_opt=security_opt + security_opt=security_opt, + ipc_mode=options.get('ipc') ) def build(self, no_cache=False): From 67957318ed5080fe2babc3b704583f9c38bd5ea8 Mon Sep 17 00:00:00 2001 From: Lachlan Pease Date: Sun, 6 Sep 2015 12:16:12 +1000 Subject: [PATCH 147/337] Added IPC spec to fields_schema.json Signed-off-by: Lachlan Pease --- compose/config/fields_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 2a122b7a31..d25b3fa236 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -59,6 +59,7 @@ "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "hostname": {"type": "string"}, "image": {"type": "string"}, + "ipc": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, From d83d6306c94c45469f86b7ae08089b8b77514be4 Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Wed, 12 Aug 2015 23:16:42 +0100 Subject: [PATCH 148/337] Use custom container name in logs. Fixes #1851 Signed-off-by: Karol Duleba --- compose/container.py | 8 +++++++- tests/unit/container_test.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/compose/container.py b/compose/container.py index f2d8a403e1..51b6258909 100644 --- a/compose/container.py +++ b/compose/container.py @@ -6,6 +6,7 @@ from functools import reduce import six from .const import LABEL_CONTAINER_NUMBER +from .const import LABEL_PROJECT from .const import LABEL_SERVICE @@ -70,7 +71,12 @@ class Container(object): @property def name_without_project(self): - return '{0}_{1}'.format(self.service, self.number) + project = self.labels.get(LABEL_PROJECT) + + if self.name.startswith('{0}_{1}'.format(project, self.service)): + return '{0}_{1}'.format(self.service, self.number) + else: + return self.name @property def number(self): diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 5637330cd4..5f7bf1ea7e 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -83,9 +83,15 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.name, "composetest_db_1") def test_name_without_project(self): + self.container_dict['Name'] = "/composetest_web_7" container = Container(None, self.container_dict, has_been_inspected=True) self.assertEqual(container.name_without_project, "web_7") + def test_name_without_project_custom_container_name(self): + self.container_dict['Name'] = "/custom_name_of_container" + container = Container(None, self.container_dict, has_been_inspected=True) + self.assertEqual(container.name_without_project, "custom_name_of_container") + def test_inspect_if_not_inspected(self): mock_client = mock.create_autospec(docker.Client) container = Container(mock_client, dict(Id="the_id")) From 866979c57bae31d42c3092bdb089a8b11907e4c6 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 7 Sep 2015 17:26:38 +0100 Subject: [PATCH 149/337] Allow entrypoint to be a list or string Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 2 +- tests/unit/config_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index f03ef7110c..a82dd397db 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -32,7 +32,7 @@ "dns_search": {"$ref": "#/definitions/string_or_list"}, "dockerfile": {"type": "string"}, "domainname": {"type": "string"}, - "entrypoint": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "entrypoint": {"$ref": "#/definitions/string_or_list"}, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": { diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 3f602fb593..aeebc049f6 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -254,6 +254,21 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual(service[0]['expose'], expose) + def test_valid_config_oneof_string_or_list(self): + entrypoint_values = [["sh"], "sh"] + for entrypoint in entrypoint_values: + service = config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'entrypoint': entrypoint + }}, + 'working_dir', + 'filename.yml' + ) + ) + self.assertEqual(service[0]['entrypoint'], entrypoint) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From b33dd3bc01a6552a8803b0c4379eb27a1b761a51 Mon Sep 17 00:00:00 2001 From: "Lucas N. Munhoz" Date: Tue, 8 Sep 2015 09:53:10 -0300 Subject: [PATCH 150/337] Fix error message and class name from Boot2Docker to DockerMachine Signed-off-by: Lucas N. Munhoz --- compose/cli/command.py | 2 +- compose/cli/errors.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 67176df271..ca2d96ea6c 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -41,7 +41,7 @@ class Command(DocoptCommand): else: raise errors.DockerNotFoundGeneric() elif call_silently(['which', 'boot2docker']) == 0: - raise errors.ConnectionErrorBoot2Docker() + raise errors.ConnectionErrorDockerMachine() else: raise errors.ConnectionErrorGeneric(self.get_client().base_url) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 0569c1a0dd..244897f8ab 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -40,10 +40,10 @@ class DockerNotFoundGeneric(UserError): """) -class ConnectionErrorBoot2Docker(UserError): +class ConnectionErrorDockerMachine(UserError): def __init__(self): - super(ConnectionErrorBoot2Docker, self).__init__(""" - Couldn't connect to Docker daemon - you might need to run `boot2docker up`. + super(ConnectionErrorDockerMachine, self).__init__(""" + Couldn't connect to Docker daemon - you might need to run `docker-machine start default`. """) From ad46757bafed98058f244960ad21edeadcd8b255 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 8 Sep 2015 19:44:25 -0400 Subject: [PATCH 151/337] Add more github label areas. Signed-off-by: Daniel Nephin --- .pre-commit-config.yaml | 1 + project/ISSUE-TRIAGE.md | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 832be6ab8d..8913a05fd2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,3 +16,4 @@ sha: 3d86483455ab5bd06cc1069fdd5ac57be5463f10 hooks: - id: reorder-python-imports + language_version: 'python2.7' diff --git a/project/ISSUE-TRIAGE.md b/project/ISSUE-TRIAGE.md index bcedbb4359..58312a6037 100644 --- a/project/ISSUE-TRIAGE.md +++ b/project/ISSUE-TRIAGE.md @@ -20,13 +20,15 @@ The following labels are provided in additional to the standard labels: Most issues should fit into one of the following functional areas: -| Area | -|-------------| -| area/build | -| area/cli | -| area/config | -| area/logs | -| area/run | -| area/scale | -| area/tests | -| area/up | +| Area | +|----------------| +| area/build | +| area/cli | +| area/config | +| area/logs | +| area/packaging | +| area/run | +| area/scale | +| area/tests | +| area/up | +| area/volumes | From 7223d5cee06b434486e83d283df198f1b62edf93 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 9 Sep 2015 16:23:54 +0100 Subject: [PATCH 152/337] Remove mistaken field detach is a run param, not a config param. Oops, sorry! Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index a82dd397db..6c73a8f315 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -26,7 +26,6 @@ ] }, "cpuset": {"type": "string"}, - "detach": {"type": "boolean"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, From 4bed5de291307f4099b6abb564bc8c2cf472dbc3 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 9 Sep 2015 16:04:27 +0100 Subject: [PATCH 153/337] Remove item unique constraint for command The command value can be a list, which would be a Unix command-line invocation broken up into individual values, thus needing the ability to have non unique values. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 7 ++++++- tests/unit/config_test.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index a82dd397db..bc033f2d8e 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -17,7 +17,12 @@ "build": {"type": "string"}, "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "command": {"$ref": "#/definitions/string_or_list"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, "container_name": {"type": "string"}, "cpu_shares": { "oneOf": [ diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index aeebc049f6..9d67a89177 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -183,7 +183,7 @@ class ConfigTest(unittest.TestCase): ) def test_invalid_list_of_strings_format(self): - expected_error_msg = "'command' contains an invalid type, valid types are string or list of strings" + expected_error_msg = "'command' contains an invalid type, valid types are string or array" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( From a372275c6ec74eb96c4d94e9821d8811caad5bba Mon Sep 17 00:00:00 2001 From: Nick H Date: Tue, 1 Sep 2015 12:40:24 -0600 Subject: [PATCH 154/337] Allow for user relative paths '~/' in a path currently doesnt work, you get the following error: [Errno 2] No such file or directory: u'/home/USER/folder/~/some/path/.yml' Signed-off-by: Nick H --- compose/config/config.py | 2 +- tests/unit/config_test.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index cfa8086f09..d5ee486bf7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -499,7 +499,7 @@ def split_label(label): def expand_path(working_dir, path): - return os.path.abspath(os.path.join(working_dir, path)) + return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path))) def to_list(value): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index e488ceb527..ddcad76e40 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -950,6 +950,27 @@ class ExtendsTest(unittest.TestCase): load_from_filename('tests/fixtures/extends/nonexistent-service.yml') +class ExpandPathTest(unittest.TestCase): + working_dir = '/home/user/somedir' + + def test_expand_path_normal(self): + result = config.expand_path(self.working_dir, 'myfile') + self.assertEqual(result, self.working_dir + '/' + 'myfile') + + def test_expand_path_absolute(self): + abs_path = '/home/user/otherdir/somefile' + result = config.expand_path(self.working_dir, abs_path) + self.assertEqual(result, abs_path) + + def test_expand_path_with_tilde(self): + test_path = '~/otherdir/somefile' + with mock.patch.dict(os.environ): + os.environ['HOME'] = user_path = '/home/user/' + result = config.expand_path(self.working_dir, test_path) + + self.assertEqual(result, user_path + 'otherdir/somefile') + + class BuildPathTest(unittest.TestCase): def setUp(self): self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') From 851129de6c38be79c3c065104a2a369d74a8dc2b Mon Sep 17 00:00:00 2001 From: Lachlan Pease Date: Thu, 10 Sep 2015 23:34:07 +1000 Subject: [PATCH 155/337] Added documentation for IPC config Signed-off-by: Lachlan Pease --- docs/yml.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/yml.md b/docs/yml.md index 3ece026494..0524940f1f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -373,7 +373,7 @@ Override the default labeling scheme for each container. - label:user:USER - label:role:ROLE -### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only, volume\_driver +### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, ipc, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only, volume\_driver Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. @@ -394,6 +394,8 @@ Each of these is a single value, analogous to its memswap_limit: 2000000000 privileged: true + ipc: host + restart: always stdin_open: true From 860b304f4afd094b3c0ffbb6964e854f79e7b582 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Sep 2015 16:30:09 -0400 Subject: [PATCH 156/337] Add COMPOSE_API_VERSION to the docs Signed-off-by: Daniel Nephin --- docs/reference/overview.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 52598737d6..f5d778fd13 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -31,6 +31,26 @@ Setting this is optional. If you do not set this, the `COMPOSE_PROJECT_NAME` def Specify the file containing the compose configuration. If not provided, Compose looks for a file named `docker-compose.yml` in the current directory and then each parent directory in succession until a file by that name is found. +### COMPOSE\_API\_VERSION + +The Docker API only supports requests from clients which report a specific +version. If you receive a `client and server don't have same version error` using +`docker-compose`, you can workaround this error by setting this environment +variable. Set the version value to match the server version. + +Setting this variable is intended as a workaround for situations where you need +to run temporarily with a mismatch between the client and server version. For +example, if you can upgrade the client but need to wait to upgrade the server. + +Running with this variable set and a known mismatch does prevent some Docker +features from working properly. The exact features that fail would depend on the +Docker client and server versions. For this reason, running with this variable +set is only intended as a workaround and it is not officially supported. + +If you run into problems running with this set, resolve the mismatch through +upgrade and remove this setting to see if your problems resolve before notifying +support. + ### DOCKER\_HOST Sets the URL of the `docker` daemon. As with the Docker client, defaults to `unix:///var/run/docker.sock`. From 4bdf57ead8078e9737c87c88b0910d5fa471938b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Sep 2015 16:55:48 -0400 Subject: [PATCH 157/337] Add a where to go next section to the main index page for compose Signed-off-by: Daniel Nephin --- docs/index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/index.md b/docs/index.md index 4e4f58dafd..72de04d342 100644 --- a/docs/index.md +++ b/docs/index.md @@ -222,3 +222,14 @@ like-minded individuals, we have a number of open channels for communication. * To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/project/get-help/). + +## Where to go next + +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) +- [Command line reference](/reference) +- [Yaml file reference](yml.md) +- [Compose environment variables](env.md) +- [Compose command line completion](completion.md) From 1eb925ee3198596b44f67fe588ca6cfdff9c9521 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Sep 2015 16:59:04 -0400 Subject: [PATCH 158/337] Link between pages in the CLI reference section Signed-off-by: Daniel Nephin --- docs/reference/docker-compose.md | 5 +++++ docs/reference/index.md | 5 +++++ docs/reference/overview.md | 12 +++++++----- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 6c46b31d18..b43055fbea 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -55,3 +55,8 @@ used all paths in the configuration are relative to the current working directory. Each configuration can has a project name. If you supply a `-p` flag, you can specify a project name. If you don't specify the flag, Compose uses the current directory name. + +## Where to go next + +* [CLI environment variables](overview.md) +* [Command line reference](index.md) diff --git a/docs/reference/index.md b/docs/reference/index.md index e7a07b09aa..7a1fb9b444 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -27,3 +27,8 @@ The following pages describe the usage information for the [docker-compose](/ref * [rm](/reference/rm.md) * [scale](/reference/scale.md) * [stop](/reference/stop.md) + +## Where to go next + +* [CLI environment variables](overview.md) +* [docker-compose Command](docker-compose.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index f5d778fd13..002607118d 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -14,6 +14,13 @@ weight=-2 This section describes the subcommands you can use with the `docker-compose` command. You can run subcommand against one or more services. To run against a specific service, you supply the service name from your compose configuration. If you do not specify the service name, the command runs against all the services in your configuration. + +## Commands + +* [docker-compose Command](docker-compose.md) +* [CLI Reference](index.md) + + ## Environment Variables Several environment variables are available for you to configure the Docker Compose command-line behaviour. @@ -70,11 +77,6 @@ Configures the time (in seconds) a request to the Docker daemon is allowed to ha it failed. Defaults to 60 seconds. - - - - - ## Compose documentation - [User guide](/) From e80f0bdf86225d172025ac70280eaf869518f5b8 Mon Sep 17 00:00:00 2001 From: Christophe Labouisse Date: Thu, 10 Sep 2015 23:41:22 +0200 Subject: [PATCH 159/337] Fix type for `tty` & `stdin_open` Signed-off-by: Christophe Labouisse --- compose/config/fields_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 38dfd2e368..5c7322517d 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -111,8 +111,8 @@ "read_only": {"type": "boolean"}, "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "stdin_open": {"type": "string"}, - "tty": {"type": "string"}, + "stdin_open": {"type": "boolean"}, + "tty": {"type": "boolean"}, "user": {"type": "string"}, "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "volume_driver": {"type": "string"}, From 4641d4052640ec541ac3fbedae5e2e5e86385b34 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Sep 2015 15:54:51 -0400 Subject: [PATCH 160/337] Document limitation of other log drivers. Signed-off-by: Daniel Nephin --- docs/yml.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 3ece026494..9c1ffa07a4 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -289,11 +289,10 @@ Because Docker container names must be unique, you cannot scale a service beyond 1 container if you have specified a custom name. Attempting to do so results in an error. -### log driver +### log_driver -Specify a logging driver for the service's containers, as with the ``--log-driver`` option for docker run ([documented here](http://docs.docker.com/reference/run/#logging-drivers-log-driver)). - -Allowed values are currently ``json-file``, ``syslog`` and ``none``. The list will change over time as more drivers are added to the Docker engine. +Specify a logging driver for the service's containers, as with the ``--log-driver`` +option for docker run ([documented here](https://docs.docker.com/reference/logging/overview/)). The default value is json-file. @@ -301,6 +300,12 @@ The default value is json-file. log_driver: "syslog" log_driver: "none" +> **Note:** Only the `json-file` driver makes the logs available directly from +> `docker-compose up` and `docker-compose logs`. Using any other driver will not +> print any logs. + +### log_opt + Specify logging options with `log_opt` for the logging driver, as with the ``--log-opt`` option for `docker run`. Logging options are key value pairs. An example of `syslog` options: From 413b76e2285f5f6575601385f9b4af7503b41c3e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Sep 2015 17:53:36 -0400 Subject: [PATCH 161/337] Fix warning message when a container uses a non-json log driver Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 48 ++++++++++++-------- compose/cli/main.py | 8 +--- compose/container.py | 9 ++++ tests/unit/cli/log_printer_test.py | 73 ++++++++++++++++++------------ 4 files changed, 84 insertions(+), 54 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index c2fcc54fdd..49071dd4ae 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -13,9 +13,9 @@ from compose import utils class LogPrinter(object): - def __init__(self, containers, attach_params=None, output=sys.stdout, monochrome=False): + # TODO: move logic to run + def __init__(self, containers, output=sys.stdout, monochrome=False): self.containers = containers - self.attach_params = attach_params or {} self.prefix_width = self._calculate_prefix_width(containers) self.generators = self._make_log_generators(monochrome) self.output = utils.get_output_stream(output) @@ -25,6 +25,7 @@ class LogPrinter(object): for line in mux.loop(): self.output.write(line) + # TODO: doesn't use self, remove from class def _calculate_prefix_width(self, containers): """ Calculate the maximum width of container names so we can make the log @@ -56,14 +57,10 @@ class LogPrinter(object): def _make_log_generator(self, container, color_fn): prefix = color_fn(self._generate_prefix(container)) - # Attach to container before log printer starts running - line_generator = split_buffer(self._attach(container), u'\n') - for line in line_generator: - yield prefix + line - - exit_code = container.wait() - yield color_fn("%s exited with code %s\n" % (container.name, exit_code)) + if container.has_api_logs: + return build_log_generator(container, prefix, color_fn) + return build_no_log_generator(container, prefix, color_fn) def _generate_prefix(self, container): """ @@ -73,12 +70,27 @@ class LogPrinter(object): padding = ' ' * (self.prefix_width - len(name)) return ''.join([name, padding, ' | ']) - def _attach(self, container): - params = { - 'stdout': True, - 'stderr': True, - 'stream': True, - } - params.update(self.attach_params) - params = dict((name, 1 if value else 0) for (name, value) in list(params.items())) - return container.attach(**params) + +def build_no_log_generator(container, prefix, color_fn): + """Return a generator that prints a warning about logs and waits for + container to exit. + """ + yield "{} WARNING: no logs are available with the '{}' log driver\n".format( + prefix, + container.log_driver) + yield color_fn(wait_on_exit(container)) + + +def build_log_generator(container, prefix, color_fn): + # Attach to container before log printer starts running + stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) + line_generator = split_buffer(stream, u'\n') + + for line in line_generator: + yield prefix + line + yield color_fn(wait_on_exit(container)) + + +def wait_on_exit(container): + exit_code = container.wait() + return "%s exited with code %s\n" % (container.name, exit_code) diff --git a/compose/cli/main.py b/compose/cli/main.py index a7b9181680..61461ae7be 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -193,7 +193,7 @@ class TopLevelCommand(Command): monochrome = options['--no-color'] print("Attaching to", list_containers(containers)) - LogPrinter(containers, attach_params={'logs': True}, monochrome=monochrome).run() + LogPrinter(containers, monochrome=monochrome).run() def pause(self, project, options): """ @@ -602,11 +602,7 @@ def convergence_strategy_from_opts(options): def build_log_printer(containers, service_names, monochrome): if service_names: containers = [c for c in containers if c.service in service_names] - - return LogPrinter( - containers, - attach_params={"logs": True}, - monochrome=monochrome) + return LogPrinter(containers, monochrome=monochrome) def attach_to_logs(project, log_printer, service_names, timeout): diff --git a/compose/container.py b/compose/container.py index 51b6258909..28af093d76 100644 --- a/compose/container.py +++ b/compose/container.py @@ -137,6 +137,15 @@ class Container(object): def is_paused(self): return self.get('State.Paused') + @property + def log_driver(self): + return self.get('HostConfig.LogConfig.Type') + + @property + def has_api_logs(self): + log_type = self.log_driver + return not log_type or log_type == 'json-file' + def get(self, key): """Return a value from the container or None if the value is not set. diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index d8fbf94b9a..2c91689807 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -1,20 +1,31 @@ from __future__ import absolute_import from __future__ import unicode_literals -import os - +import mock import six from compose.cli.log_printer import LogPrinter +from compose.cli.log_printer import wait_on_exit +from compose.container import Container from tests import unittest +def build_mock_container(reader): + return mock.Mock( + spec=Container, + name='myapp_web_1', + name_without_project='web_1', + has_api_logs=True, + attach=reader, + wait=mock.Mock(return_value=0), + ) + + class LogPrinterTest(unittest.TestCase): def get_default_output(self, monochrome=False): def reader(*args, **kwargs): yield b"hello\nworld" - - container = MockContainer(reader) + container = build_mock_container(reader) output = run_log_printer([container], monochrome=monochrome) return output @@ -38,37 +49,39 @@ class LogPrinterTest(unittest.TestCase): def reader(*args, **kwargs): yield glyph.encode('utf-8') + b'\n' - container = MockContainer(reader) + container = build_mock_container(reader) output = run_log_printer([container]) if six.PY2: output = output.decode('utf-8') self.assertIn(glyph, output) + def test_wait_on_exit(self): + exit_status = 3 + mock_container = mock.Mock( + spec=Container, + name='cname', + wait=mock.Mock(return_value=exit_status)) + + expected = '{} exited with code {}\n'.format(mock_container.name, exit_status) + self.assertEqual(expected, wait_on_exit(mock_container)) + + def test_generator_with_no_logs(self): + mock_container = mock.Mock( + spec=Container, + has_api_logs=False, + log_driver='none', + name_without_project='web_1', + wait=mock.Mock(return_value=0)) + + output = run_log_printer([mock_container]) + self.assertIn( + "WARNING: no logs are available with the 'none' log driver\n", + output + ) + def run_log_printer(containers, monochrome=False): - r, w = os.pipe() - reader, writer = os.fdopen(r, 'r'), os.fdopen(w, 'w') - printer = LogPrinter(containers, output=writer, monochrome=monochrome) - printer.run() - writer.close() - return reader.read() - - -class MockContainer(object): - def __init__(self, reader): - self._reader = reader - - @property - def name(self): - return 'myapp_web_1' - - @property - def name_without_project(self): - return 'web_1' - - def attach(self, *args, **kwargs): - return self._reader() - - def wait(self, *args, **kwargs): - return 0 + output = six.StringIO() + LogPrinter(containers, output=output, monochrome=monochrome).run() + return output.getvalue() From 7d8ae9aa6dc907975433657a926c0e329d937d41 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Sep 2015 18:26:34 -0400 Subject: [PATCH 162/337] Refactor LogPrinter to make it immutable and remove logic from the constructor. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 93 +++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 51 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 49071dd4ae..845f799b79 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,8 +4,6 @@ from __future__ import unicode_literals import sys from itertools import cycle -from six import next - from . import colors from .multiplexer import Multiplexer from .utils import split_buffer @@ -13,82 +11,75 @@ from compose import utils class LogPrinter(object): - # TODO: move logic to run + """Print logs from many containers to a single output stream.""" + def __init__(self, containers, output=sys.stdout, monochrome=False): self.containers = containers - self.prefix_width = self._calculate_prefix_width(containers) - self.generators = self._make_log_generators(monochrome) self.output = utils.get_output_stream(output) + self.monochrome = monochrome def run(self): - mux = Multiplexer(self.generators) - for line in mux.loop(): + if not self.containers: + return + + prefix_width = max_name_width(self.containers) + generators = list(self._make_log_generators(self.monochrome, prefix_width)) + for line in Multiplexer(generators).loop(): self.output.write(line) - # TODO: doesn't use self, remove from class - def _calculate_prefix_width(self, containers): - """ - Calculate the maximum width of container names so we can make the log - prefixes line up like so: - - db_1 | Listening - web_1 | Listening - """ - prefix_width = 0 - for container in containers: - prefix_width = max(prefix_width, len(container.name_without_project)) - return prefix_width - - def _make_log_generators(self, monochrome): - color_fns = cycle(colors.rainbow()) - generators = [] - + def _make_log_generators(self, monochrome, prefix_width): def no_color(text): return text - for container in self.containers: - if monochrome: - color_fn = no_color - else: - color_fn = next(color_fns) - generators.append(self._make_log_generator(container, color_fn)) + if monochrome: + color_funcs = cycle([no_color]) + else: + color_funcs = cycle(colors.rainbow()) - return generators - - def _make_log_generator(self, container, color_fn): - prefix = color_fn(self._generate_prefix(container)) - - if container.has_api_logs: - return build_log_generator(container, prefix, color_fn) - return build_no_log_generator(container, prefix, color_fn) - - def _generate_prefix(self, container): - """ - Generate the prefix for a log line without colour - """ - name = container.name_without_project - padding = ' ' * (self.prefix_width - len(name)) - return ''.join([name, padding, ' | ']) + for color_func, container in zip(color_funcs, self.containers): + generator_func = get_log_generator(container) + prefix = color_func(build_log_prefix(container, prefix_width)) + yield generator_func(container, prefix, color_func) -def build_no_log_generator(container, prefix, color_fn): +def build_log_prefix(container, prefix_width): + return container.name_without_project.ljust(prefix_width) + ' | ' + + +def max_name_width(containers): + """Calculate the maximum width of container names so we can make the log + prefixes line up like so: + + db_1 | Listening + web_1 | Listening + """ + return max(len(container.name_without_project) for container in containers) + + +def get_log_generator(container): + if container.has_api_logs: + return build_log_generator + return build_no_log_generator + + +def build_no_log_generator(container, prefix, color_func): """Return a generator that prints a warning about logs and waits for container to exit. """ yield "{} WARNING: no logs are available with the '{}' log driver\n".format( prefix, container.log_driver) - yield color_fn(wait_on_exit(container)) + yield color_func(wait_on_exit(container)) -def build_log_generator(container, prefix, color_fn): +def build_log_generator(container, prefix, color_func): # Attach to container before log printer starts running stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) line_generator = split_buffer(stream, u'\n') for line in line_generator: yield prefix + line - yield color_fn(wait_on_exit(container)) + yield color_func(wait_on_exit(container)) def wait_on_exit(container): From 61415cd8bcf2d08dbfea957ca4801ed4f0f6e554 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Sep 2015 14:38:46 -0400 Subject: [PATCH 163/337] Fixes #1955 - Handle unexpected errors, but don't ignore background threads. Signed-off-by: Daniel Nephin --- compose/utils.py | 26 ++++++++++++++++---------- tests/integration/service_test.py | 6 +++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index 690c5ffd53..e0304ba506 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -21,7 +21,6 @@ def parallel_execute(objects, obj_callable, msg_index, msg): """ stream = get_output_stream(sys.stdout) lines = [] - errors = {} for obj in objects: write_out_msg(stream, lines, msg_index(obj), msg) @@ -29,16 +28,17 @@ def parallel_execute(objects, obj_callable, msg_index, msg): q = Queue() def inner_execute_function(an_callable, parameter, msg_index): + error = None try: result = an_callable(parameter) except APIError as e: - errors[msg_index] = e.explanation + error = e.explanation result = "error" except Exception as e: - errors[msg_index] = e + error = e result = 'unexpected_exception' - q.put((msg_index, result)) + q.put((msg_index, result, error)) for an_object in objects: t = Thread( @@ -49,15 +49,17 @@ def parallel_execute(objects, obj_callable, msg_index, msg): t.start() done = 0 + errors = {} total_to_execute = len(objects) while done < total_to_execute: try: - msg_index, result = q.get(timeout=1) + msg_index, result, error = q.get(timeout=1) if result == 'unexpected_exception': - raise errors[msg_index] + errors[msg_index] = result, error if result == 'error': + errors[msg_index] = result, error write_out_msg(stream, lines, msg_index, msg, status='error') else: write_out_msg(stream, lines, msg_index, msg) @@ -65,10 +67,14 @@ def parallel_execute(objects, obj_callable, msg_index, msg): except Empty: pass - if errors: - stream.write("\n") - for error in errors: - stream.write("ERROR: for {} {} \n".format(error, errors[error])) + if not errors: + return + + stream.write("\n") + for msg_index, (result, error) in errors.items(): + stream.write("ERROR: for {} {} \n".format(msg_index, error)) + if result == 'unexpected_exception': + raise error def get_output_stream(stream): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index b6257821dc..79188f69a9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -638,8 +638,7 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(service.containers()[0].is_running) self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_scale_with_api_returns_unexpected_exception(self, mock_stdout): + def test_scale_with_api_returns_unexpected_exception(self): """ Test that when scaling if the API returns an error, that is not of type APIError, that error is re-raised. @@ -650,7 +649,8 @@ class ServiceTest(DockerClientTestCase): with mock.patch( 'compose.container.Container.create', - side_effect=ValueError("BOOM")): + side_effect=ValueError("BOOM") + ): with self.assertRaises(ValueError): service.scale(3) From 1fcacae1fe381331a324399332bdb8b0fa926a1a Mon Sep 17 00:00:00 2001 From: Luiz Geron Date: Fri, 11 Sep 2015 19:06:08 -0300 Subject: [PATCH 164/337] Fix schema.json MANIFEST.in entry docker-compose up doesn't run after install without this. Signed-off-by: Luiz Geron --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7d48d347a8..43ae06d3e2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md -include compose/config/schema.json +include compose/config/*.json recursive-include contrib/completion * recursive-include tests * global-exclude *.pyc From d1dd06a7e2f28c0e2f7df53c0a7af13d6b12ece1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 11 Sep 2015 11:24:57 -0700 Subject: [PATCH 165/337] Update docker-py to 1.4.0 Signed-off-by: Aanand Prasad --- requirements.txt | 2 +- setup.py | 2 +- tests/integration/service_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 666efcd268..4f2ea9d14f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.3.1 +docker-py==1.4.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/setup.py b/setup.py index 29c5299e9d..0313fbd052 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.3.1, < 1.4', + 'docker-py >= 1.4.0, < 2', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 79188f69a9..bb30da1a1b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -864,7 +864,7 @@ class ServiceTest(DockerClientTestCase): def test_log_drive_invalid(self): service = self.create_service('web', log_driver='xxx') - self.assertRaises(ValueError, lambda: create_and_start_container(service)) + self.assertRaises(APIError, lambda: create_and_start_container(service)) def test_log_drive_empty_default_jsonfile(self): service = self.create_service('web') From a95ac0f0e0c973a139a50c1e0af10055e32f829a Mon Sep 17 00:00:00 2001 From: Charles Chan Date: Tue, 1 Sep 2015 21:11:43 -0700 Subject: [PATCH 166/337] Touch up documentation for Docker Compose. index.md: * clarify Python & Flask * minor edits & reformatting install.md: * merge the elevated installation instructions with `sudo -i` discussed by @asveepay and @aanand in PR #1201; fixes #1081 (not sure what happened to the merge, but it's not showing up on the master branch or website) * minor edits Signed-off-by: Charles Chan --- docs/index.md | 20 ++++++++------------ docs/install.md | 17 ++++++++--------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4342b3686d..992610a980 100644 --- a/docs/index.md +++ b/docs/index.md @@ -74,7 +74,7 @@ Next, you'll want to make a directory for the project: $ mkdir composetest $ cd composetest -Inside this directory, create `app.py`, a simple web app that uses the Flask +Inside this directory, create `app.py`, a simple Python web app that uses the Flask framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): from flask import Flask @@ -113,12 +113,12 @@ This tells Docker to: * Build an image starting with the Python 2.7 image. * Add the current directory `.` into the path `/code` in the image. * Set the working directory to `/code`. -* Install your Python dependencies. +* Install the Python dependencies. * Set the default command for the container to `python app.py` For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). -You can test that this builds by running `docker build -t web .`. +You can build the image by running `docker build -t web .`. ### Define services @@ -135,18 +135,14 @@ Next, define a set of services using `docker-compose.yml`: redis: image: redis -This defines two services: - -#### web +This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Connects the web container to the Redis service via a link. -* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. +* Mounts the current directory on the host to ``/code` inside the container allowing you to modify the code without having to rebuild the image. +* Links the web container to the Redis service. -#### redis - -* Uses the public [Redis](https://registry.hub.docker.com/_/redis/) image which gets pulled from the Docker Hub registry. +The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. ### Build and run your app with Compose @@ -163,7 +159,7 @@ Now, when you run `docker-compose up`, Compose will pull a Redis image, build an If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. -If you're not using Boot2docker and are on linux, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try localhost:5000. +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try http://localhost:5000. You should get a message in your browser saying: diff --git a/docs/install.md b/docs/install.md index 85060ce040..371d0a903f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -16,16 +16,11 @@ You can run Compose on OS X and 64-bit Linux. It is currently not supported on the Windows operating system. To install Compose, you'll need to install Docker first. -Depending on how your system is configured, you may require `sudo` access to -install Compose. If your system requires `sudo`, you will receive "Permission -denied" errors when installing Compose. If this is the case for you, preface the -install commands with `sudo` to install. - To install Compose, do the following: 1. Install Docker Engine version 1.7.1 or greater: - * Mac OS X installation (installs both Engine and Compose) + * Mac OS X installation (Toolbox installation includes both Engine and Compose) * Ubuntu installation @@ -33,9 +28,13 @@ To install Compose, do the following: 2. Mac OS X users are done installing. Others should continue to the next step. -3. Go to the repository release page. +3. Go to the Compose repository release page on GitHub. -4. Enter the `curl` command in your terminal. +4. Follow the instructions from the release page and run the `curl` command in your terminal. + + > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory + probably isn't writable and you'll need to install Compose as the superuser. Run + `sudo -i`, then the two commands below, then `exit`. The command has the following format: @@ -69,7 +68,7 @@ to preserve) you can migrate them with the following command: $ docker-compose migrate-to-labels -Alternatively, if you're not worried about keeping them, you can remove them &endash; +Alternatively, if you're not worried about keeping them, you can remove them. Compose will just create new ones. $ docker rm -f -v myapp_web_1 myapp_db_1 ... From 32cd404c8c1bb31d04aa2845ca7fa15deae9b00b Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 8 Sep 2015 11:54:03 +0100 Subject: [PATCH 167/337] Remove redundant oneOf definitions For simple definitions where a field can be multiple types, we can specify the allowed types in an array. It's simpler and clearer. This is only applicable to *simple* definitions, like number, string, list, object without any other constraints. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 5c7322517d..6277b57d69 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -24,12 +24,7 @@ ] }, "container_name": {"type": "string"}, - "cpu_shares": { - "oneOf": [ - {"type": "number"}, - {"type": "string"} - ] - }, + "cpu_shares": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, @@ -74,18 +69,8 @@ "log_opt": {"type": "object"}, "mac_address": {"type": "string"}, - "mem_limit": { - "oneOf": [ - {"type": "number"}, - {"type": "string"} - ] - }, - "memswap_limit": { - "oneOf": [ - {"type": "number"}, - {"type": "string"} - ] - }, + "mem_limit": {"type": ["number", "string"]}, + "memswap_limit": {"type": ["number", "string"]}, "name": {"type": "string"}, "net": {"type": "string"}, "pid": {"type": ["string", "null"]}, From 418ec5336b0c7848087b38b3d2617b2ce340c67d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 8 Sep 2015 12:01:47 +0100 Subject: [PATCH 168/337] Improved messaging for simple type validator English language is a tricky old thing and I've pulled out the validator type parsing so that we can prefix our validator types with the correct article, 'an' or 'a'. Doing a bit of extra hard work to ensure the error message is clear and well constructed english. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 41 ++++++++++++++++++++++++++++++------ tests/unit/config_test.py | 15 +++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 632bdf03bd..44763fda3f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -117,6 +117,38 @@ def process_errors(errors, service_name=None): else: return str(schema['type']) + def _parse_valid_types_from_validator(validator): + """ + A validator value can be either an array of valid types or a string of + a valid type. Parse the valid types and prefix with the correct article. + """ + pre_msg_type_prefix = "a" + last_msg_type_prefix = "a" + types_requiring_an = ["array", "object"] + + if isinstance(validator, list): + last_type = validator.pop() + types_from_validator = ", ".join(validator) + + if validator[0] in types_requiring_an: + pre_msg_type_prefix = "an" + + if last_type in types_requiring_an: + last_msg_type_prefix = "an" + + msg = "{} {} or {} {}".format( + pre_msg_type_prefix, + types_from_validator, + last_msg_type_prefix, + last_type + ) + else: + if validator in types_requiring_an: + pre_msg_type_prefix = "an" + msg = "{} {}".format(pre_msg_type_prefix, validator) + + return msg + root_msgs = [] invalid_keys = [] required = [] @@ -176,19 +208,16 @@ def process_errors(errors, service_name=None): service_name, config_key, valid_type_msg) ) elif error.validator == 'type': - msg = "a" - if error.validator_value == "array": - msg = "an" + msg = _parse_valid_types_from_validator(error.validator_value) if len(error.path) > 0: config_key = " ".join(["'%s'" % k for k in error.path]) type_errors.append( "Service '{}' configuration key {} contains an invalid " - "type, it should be {} {}".format( + "type, it should be {}".format( service_name, config_key, - msg, - error.validator_value)) + msg)) else: root_msgs.append( "Service '{}' doesn\'t have any configuration options. " diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 870adcf81e..90d7a6a26d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -269,6 +269,21 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual(service[0]['entrypoint'], entrypoint) + def test_validation_message_for_invalid_type_when_multiple_types_allowed(self): + expected_error_msg = "Service 'web' configuration key 'mem_limit' contains an invalid type, it should be a number or a string" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'mem_limit': ['incorrect'] + }}, + 'working_dir', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 0bdbb334476e6155cae346734a21f428ecc15a11 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Mon, 14 Sep 2015 23:46:48 +0200 Subject: [PATCH 169/337] include logo in README Resolves #2024 Signed-off-by: Tomas Tomecek --- README.md | 2 ++ logo.png | Bin 0 -> 39135 bytes 2 files changed, 2 insertions(+) create mode 100644 logo.png diff --git a/README.md b/README.md index 69423111e5..3c776a71c2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ Docker Compose ============== +![Docker Compose](logo.png?raw=true "Docker Compose Logo") + *(Previously known as Fig)* Compose is a tool for defining and running multi-container applications with diff --git a/logo.png b/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..9bc5eb2f9e5051601aa8628f194932c8001e26cd GIT binary patch literal 39135 zcmaf4V{j#Zu+7HKjcwbu?cLaRHg>YH?Tt3J?Tu|)8|TL6yZ=}9KEIlpnX38ptDeR= z-KQf}l%$d1@!-M0z>s8RB-B9fQP8~s3k`bG;B8leUeG4;(h^`_|6PB&OOrug;C{+z zyMTdVQT}&-lU1X*27QEal~t64*?~iW0nmUwt*e89k$}lch<^86zsT|MC0J^I>OG#E zMq)bM3cj#;0_TPS9C7V&ors)TWx#V zHrH;Ts7wdCOuvm(6{R-O^S6`sJ$}C5NUv|Lt<7D$D@3Fee73m*{!Zn{< ziJ#sJGJ`eU{J848?z5)L)tkfRjnpje7h3L@n{Yt_}%T4(J5l%vG{E9u2E?5uHUJJKCJ8S zfs!(Ts%mTs(1oIvH8o5V2RGS!H=gfeuw;p&Z{l=vkNOYAm8OOtt#4o%EDpXU#<&}f zZFjcfREn8l9E^}?=C(z~fM_XLs12`!Lu}8Z_+IkWrF9i;UgzzjsjMfZ43;$e4fhhb zq~TtoR{s|w|A!NCzuYW7^Zkg&u<4sv`zdT)p_L*Lo($3yJkeG6&PLzMnxC_`I{tc0 z*mG7XEh5;s^A4jEliqu0otM4m#g<Gh`t^HH*ZrpEpdFb)n* zuG@dg#Q(a7m`CX7!WdVZU3QY1AY~5>qZ~L-9w(R^KAWDq*(VXdrdX@2SNqV+J)Iep zk=GAM8_U1~<^WM)vBPwDvPxc1o_-ky!fECOoRBss4PQXb)sLe*Kp;6#o0swoALWyx z)Z|_(1iacU8_CI4co{EN1S|~x)^O(qg{xLy*4nABXez7KLBG!En`8Hb#%~hqcUzE6 z*m^08xMDhY4-PXAED}=FWBDB^a)SgeVh|OKNt&S5O(06pV86*Bq%2O0p+-d{t%gxG zo&v7k|8-s%-VbjudI z62$;>zY}PGu7S&&CGfgo9Wb?F=}ND?Z|gF6?*)|G)|UF7z}_N+{gIeNRn#GE9zLEa zDlANmwH2bl=XSWb3=1RAS^)WUURY!!^EA+^{L|oM{0nBS-DwM9lu<0l3CVp3$HxB& z*2nOb{3Y39Z-!B{z0lJ>P=jl95wC5yo?hV?y9qL1BdaN$ryz6oP&ehcwGhD21he@mMm*3lCF({zJQF@2oH7IMN z_1_fhCUe_DOwFut_)rhBX8#9%^s@r?M~Kg(>c>Re_7pE6swz0YNIf{JdG z14kN|;;@jrX}ss`|6s!TYg%H`N49LTytpo=ELeM<*An>N`aClzZm*W(FtwWIAG`xb zjQWO6gEHnLeLbD!aT|B(qLg`aIw}OK{0^Yfy6O|OIONO^Wf)r1OVY5GnRFFs2)1{& zw?9Syi&%A%;s-%i@7K{$>y<~!aOkFsN@h&t&^sYYkx|?$0303&?kc#mNJ))Ogc@4trSf{ks={+U)xJYoczvbx`&nt>#w~f? z!F}imB}#{%;9jcVKFQh+zlmA;MQ!zdp*;G%BbNByDp}|SFl+d8g+bDjoFKZ9t^c;F z6_nWhYk_>VCby8w=t{^ZLNY5!fhO4@SQf+<(hE7RENN*Lyb*kDLq^;cgaIxt2Whm? zl1L?k_l04%-npWuPLO)CKTA-{o^c ze#7r)RZ-0@V#22RwjO3Rn206kCXM05;DaqewtJJ$k59|Ucvy^`Tijg!ZGdgt*Yai` zX#KyTw{n1tH-7?O^+w;%wvX(GA3~T;Lt7->Ff#vF18+?WM*9U28#Lpbfe8;0CJxOH z&cyC8z?0dz>aG`5mb*lS7-?fof(3tbuO3g%C@da^5%5&#NxT5bD_U>5aw#I^wWpuS zEM(#1?+M|$%y$;Y@rou&ok$f)G1sq6#kf=^1K696AU5M488V3rVJRl3G}Ch~YxBm8 zluyn8`nJrTXi-a1{Pl_6p~8bp%1eKmsLpEb&HK-_zAs*Ge}(5CNKLVC7WR`O4D;z1+#Z0ef^ZBqE{KINRo)iBWMwT;0jkSh zZ_X|arClG(1uFmLpoN>&8N%9A7Isr_s!J8_1B|;gl415AKk~I zRumR7NC1Re8@zpn)xhB@hWn&ysY2t*^vUv!H-)sX*V;|jixx(RNXxQO`&J<9!AMSz z^}+HHzGGJ+hv@us`hH4e@@k#ShM{y!8G&kw=56OFg7ApX%J;#}FNpj$4puZ7QZxw} zNS-Ea1!l1!aZ9?RcUNs>ZNvrAm}RFWd0q$o0oMp)?YQU_wpq8%o40pEgU8E;0mGhS zd4Y#T_5Ylxb||b`7i^S~7FmP1qQ_WkC_}z0u||JboO|zc+q}+PR$wB%4Li!fN6f|y z=nxa5AWfGMHbBBiu)@@zNVb(Uz;S>!1|5Is*=hn$W&|I50jsW-(h0Xa7w&(X*y9LF zU}jnk|Lmo~By@HR>UIlwi;JODU4Uc_y)H<@g{|cHwgfrjv)yKM@Jr4aS{BHBSP<4D zl;T|m5)qK5o0rwx)+j;uzf#YyC*e)53<7nbHWX`B{RB2^T|3@ne`T^;s_E(?h*(>n z(#oBNXX&gHt@-?U#Xl%jc*=G=J>*y<_+x&D){rI~>Q{s%=uU*Q?Ku(#7|tUSNsSG< za4t=QM87CI7zV+H9C#=~v6qj8J*f0wM(q24>Hck-KO@(yspHu93Nuf9gidX^FL81a zm^}P`AU^d;cUb?KWMuV{5YNfU>BostZ;4oq(6cM=}}VY-bO@&n_lR2ucS3n`xq1^_&OR12I;^}2%;O;V~uC6LSPfYj_m zNWvBgH}NktrPls~cW$ktgMoL~Rx5|7l1dC7XOy7N?C=``?fcwk$f-iUUVdYE7yhpf;{zMWx zNLGnIgj?Q_I_H39m_OaLM-33D0bxF!^aF3R_qgm>J@4}q)nLa4&P<&7%z~xx&L2;Ui_3!W*Sq4v<1}? z)a$U8DpWfuBI(KBlMlN;-QOwD- z+J|l&Zns?LJc%LfXy^6>f{YY&(qES&-Hw*2mNkU-I_S0Cp;Ys-IqEXLfigW9D|6#>2ps~6-_V>qI zPUfK+-;li8V-D&(M3$3!iI3xRzu>XJ$FsPja+Xm1gY7~E)+yn^z`$@UekPxr&CG$< z-4Sk|(QI+^w2iejZhlI~Pz$hMg@!`_D_Rj%3=kO-Ib@Pif=Ucae_&sZSTKhp^26W% z)1ca|dg4&$%DpoWCeF;#GTd@91J&oEe#Bd7=J@W1VxG%#-PQYvC1fGII8Wnv^7QwG zs0=kb-Cmm&zt$ZVt`A@_+-<(eJrzj~WbCzyPJMhNREvmq=Z3O%0ed-kG(dLqYoutTY}z zEL3nt7ztJ^8JKdIVLQW0$9N(+|FXmC&&Lm8&k$Zq1`%6P#0^%yRH>0;6r1YAUC(yN zMa`?@&YiB~Pr(gtKkARE61ZEGzFA|;lyppO7-WKXcOU6vY>>fr*f?=B*JzMFFMR2w z;!Ga1y<&*?b2s0~8l0UIOF~OLbX$GArQNVuc>OVoFP>TP7McL{G&w*Z>M&)l7DWrsEjv;vl0iO()EYTH3MRT=AA7y+QExN*5C&cR$G9pv`+li zbM!OUS$V?u%|}NiF!LbZc@<=gv)YoplaiJ7wqZo#SuTZ5V{e2*H6R_g+r&O?Hcq>M zqsAL}aUp=w&$`%5754M9e0w!y)`<-ExGi^WKh6@K7Ls)X8nFmKlU;8Kjx>Js)1K3( z)xl^ZZHmFLC1v;(??MZal%l6fuVh9rS$XSPTPikJW>JHtIQt1zPuO2zqt)@h4p$*LJ$?A&qsFjuM#94E>`y~Y%VR;YT``)e+XR6nNwWs&TALlnw|Av^i~19r5xSun9sR)ybNRJ;dvrQ93M?elIfv=qfABTkHyY#PVLWIG z=LQ@4yw(0<`G7C~H>4&mqMnADp37Qm5KvcTwO{|fT)Q*6+3q53w_3LcQEW0KhG%AG z!Rmj;;L!2nE*Jj8h@2IvaD-%8hxZ1yxW4%hIZiP>U_b*}?%rZe?i;Q_v|cYl;2C}+ zj>?6^&PtMqb!=SNTB|90t-x0=$c!OBS$>g{kTb_eGj*oL`yF9zGUY9-OWPKu3^w(1 zAcE@o%PWxE2-fE)UmRv7JLR{S_Zl&wdWk}QPfzZQ|Kq~&(Ufj(O?9=;eB7FUD+iDH zBd+h~#LmBU)a?zGb?>dN{^@y)hA~C2ai^Vg1mcgl}GNR3Kz5~}}c>qQWVa-)~ugE3VEGCOu zMTcY+9V3`@Z0zi~1_smkwY~li5vZ5}Ha0f@+r$*ga2i)wcGR1h%^#pkXVkqhAOY~r zv~Y%oQtG|D)Ve%aI`k#$hb&{wbFdz8iN-&WWM>h?iJ96aWH@)A+2kKEM?)Huup$Ol zlRbBB2Mz@k2(wD;utFDHDT^Y~<|IzSiy9lVx+wD}!!1}Wtj&qqPm%t{vt8>jX5??N zGkFh*G&?ja{H_N?4>BgFt4BQ5eZ1W8NJpXd%P($j7E(;$@K({)oG6Q8T{w>;&_<7e zltXA}#Nl=!;@DScUtfuY~ zVSdF}Kti1y#fqNuii*UQx?XGmI4Kh= zQlE(AN!af^gu?nvBUq?ngA-;4mHe6Ab2)~^K2ezR^8qzVIf&vXSjWE= z+x_M*pdln~-c+tWyEBIkCt34HVbbKLl`t$%P{@(f`X~@aPa-v7FAgOk=MQf$&5?YW z-6L4-j9)&IYO&uM+!pl7;C10MXkBTy7?|2zP__hixLN3`AH)uXrPKsCzTD=3AVHGx zW!IYBT2tnJ66rA)v9^ZqdN)=UWQvC#$|ZJ`Y1TS*K;;5>LN*}#GGa4dO5 z2$?H1)Ovzd{yu)kiS{vHjmh7*A5#{mukAmx^#brUJSNQkI7={_do$RmYSjF$JlBtn#ntAEt+(^=diqfD_S&RVsMXadhYMR98LuCJm$-Dj0V)`!6uk(e_q;DCUWI~ihCh;=XjWdAWO<>i@jZ8WI;{l z-r^xX7O_IDm_$(+u5j*V5erJkR5sZOy<~r8#^SJovrL8mT(jC}y^8nWKP*&rmaYc0 zfUx*f;;#6`M=$YkWLpOZYwc93QMd@zLaCegyBT$D?U(ZD>1pP6#HnX?rhf-{o%p)= zH)%OV6fnOSk%|^Q5kqH=m9z%o(^Jem8Q27Z6ly6(ER%AK_nvSvN@8qG*kog+Pwalq~JaXBr}4D!;e&o$7`tW1y7Vzc>txN0T|im*UE z14SAPqw^BC)fPUV=j**0E)Z2M1_9^%_V!E=0zCwRs zE}>Ixg_Kp+BqFwh{9_o?pJM_wxuT!9H67?^>dcQ0V~G?xLL$y56!nP|YUJh7JYzcr z1aPH+<;f-F3du{AJEisYk+Jw(KQFxySQxO=%vnu`5s|`y0Fg}f6(GSIf(0I8az9uO3GuDP zaZ94DGWNA&V3ObEo0xn`fA3r~_FzRR`73}P+O1QaK7&q6l zaKW7OZBwJxwIes$0$pxRVn=N#)(sy#1$4rm+up-(*nAnp3rebF*AWtl{c#0zUBre} zm;}<)2>ApNSwT}-z2Sn+IKf=2NR|vPKjbW~s^javZjMEw0$)#FaXKkeTZs6PfmOI< z&nqDJsg)xmk&wy1Nk#ee&(mPHa~{_dP?u?hf74-QLtj7VFw;xY_XS^A!u#XC1hpiM zP=}9Ea1Jj$e{5n*0uXz(_|d;oFfxL_a+kb{)zMo@C;M2#Gd^RMbYj&q@oU5n1&c@Z z(|SQ^@53W~c5Q9E1*psWmJQX$&c-@xc6_vV=c!fy?#H*WAuarNJAvO~x7KhM!#=&c z+ZSvdL{N((r1R)onRA3dGn}w zP$YS;{-^eI3{o|w*hF9>&%Nqmaqf~#?|X%4%-N-B5+OHE{h;b*|A)-60A=KQl)7ck z^kjP}I-+a6`;d>_Mi(+EWy`#4m`QfP=?}A`e{*Zg4)*J9dtSfhNe@AVM8IT!FtSir zM~8?DDLJ~PElgdh7p!=yO^h{xiq2q@^q{yAO)+jqF>NG@o~(h|FN)&#)L++!Pa8Ef zTR)a1E0Jzk*`N?OA|^%WW(R<7i8LXG(yPF{$?ZSdu8-Hs5HWp%8s1S}Y<*En5|M*M zKE&4a`FYZd&5lGZ8ZB14!BGg9N@P0)^nW}d_8U)-5P|G$pxk?T#Lvp^EII*6I~UNJ zuGEvSk)f8Dyw`u_lJ8UTH7L@m8o)`Ds-Mv24i((#p3D_ME=lrJBtCB&;thrrMtufM zo+qjQYNnt(Ki6J&d0}a7tc>-5WlLzCpHDj872^~)*6r72zZ>;)=B8M4aGER z*d*xrM-q}6XL?}`OY#_ML#^dF;A6#LW3i^mN$nU0s1~TlChqrHky4ER6vyy;z5Pb@ zvyr?f>GWlrnKM##$(f9m7XyaNF!9FOz?@Pz?DW747sa59uzUAYSj_Ne$xuvI97e-I zcBu6C1E|6XDSORmT&CN>)Q4fAgLk^_ruqG>G0%B|CV=gnvrFwHy=^JFy`Csct;E(c zdTD`>MJ=qMQt+`O9HaLgW4Asx-g-D{{jHwKQ z{wmE%h&3G#(YElUqm%cIW^-Z%bFiV5;xtT*@rE5zq@!cgCZI|lA=Ur=#}B05hh<$u z98Mbyd~PCPsxjG(u64jG)dB}C4Q`ZhEzfoyUDyPa!$7P?S|d&oE_O3~H6|B$Ip(IQuZ`py+!bHowq0=wp&5f=vIpx3&+ zAka&u!S=%1u$gaB;+g8FvN<WioVx_R1P|*u&%LeNWqcl zRHVg`X*aM9Q9gEvfh=YMbTukB-pIH+_RR@Soch#HweHNr5-UKkC3$Ls^l^lxw``dQ z@4Tnz#W3H1u~bjvf;>q4YY)Vx2YGmS9JVGjd-1J&x(gN9d`W?sNm}SEzvUMR(c}%SlNE;nMMVsdpFbGmi|>^3&Nlx4yQ^RePg} zbOgMsRsUt1DR-%Mae_AY8UT(sULH_2XKDV!n0+{d-ZaX zb)SLb5{3GME{jEdqpGH`k|q*BmV6bkYM^J5spI*gDtZYZ*4iZTI%Q0qYDx+H-__vY z82YOK6D>`9`z)Fop3iR=+o)g} zoVHk%8oW@BiB?T2>#88`MaGDtwzfLvekAIWmV~qpD3@LOf=cX%;uqOJ2|Uxylj%}A z@g-9$%|xtYSF78#^3s75Jf~36`W}W_TYF_?I}AYESXM&v&;A!GS!{-#08iSuIB3#&NBECPyDy(oLn8X@}8nhl~w-X z)RYH+HCpcRyiCujU=#P=EA;1s-A(sI$k%k|%EYv}koO?S2M^T>j_j}GlJZlpI9TzR zlu*hDXZt;_RT=i?f>>+rJCIq>u=nYFWhmwIy?^ShBu?N!PP6X4rchdxfFO&~pZJyp*+d668NBho zP+P(}LNYc!Y#vTgugOA3(hbittyHC)VWeXdQ-$TUQl_S+?yD zaX|PKzuEueRnR(!!supyNYD3XX!+Ls>T7qqb~`|Md35c&fiKb_wm*SK$xbF~ISJku zIHJf<@)yP~hbXq7bw~?o#5XAgBYZsZ;5eET;{f8LRz&+5WiQ@o9XVZ8UucO3w5k zlAAK};hn*$v~Mw?B86YK33j`?jdR>z#u1PFNjWRLc&8Oq_{dv8wBoElS4ycGY}pA~ z9@{7JE0X&1?;&$*^;DeiXROZt)UU2)G&8p_{!P_V&7v1(8a}u~muO~Yw(sr*>*l7S zu5McVgJHK?)9~<}zE(_{T&pB|dsgMkoILi#>)C+9*1#N@l&IyBYTDVAYsI)rMfiuEm z!yX2r@j0P4Ylg=01aR=*VJBzgz%d}O8HM)^m0-(4Fdl=Sq-5Zuy3s> zE1|8V#0FtGnc<9(AwGoy^%;=-Q7Lqb=XmxZ}gemn>#sv1{86yun~hdDbt`FUArtdd}O@%=meNYDkQ`}8Au(SkE4 z05;;F*l#&pD&fI)SzXP*J?j3IsrMzWb1p2aYm*>?w#itOBMMo%E;@{q>6v4WeCfNR z&Ul_Mcix`MjQJUG$9Ocv8ZrcdB13H6`%hGNzy}gP)(7+49(ttjBWi{#6KOAh4gYDN zcaw1tiz?ag-3-)NkORor1C3J}=H&RRmy)5+CC&;8Or3J5HV%`4XXW3il6k-souR^J6Rv_tQoWmShv0hBd z3Smv9n}9`{?4(ApPga~%#3#a8#t-boO*ej;KG!bBQ&s?ZrWucplSajTO2|$sG%W|c zuvVP?hK#zJ%6E*b@};YWRCJDuOSqC0G|*(0np<=@j*!!7=QDVKN|6tfpDJ;@q^xL= z);f0BuS4utjED$7{U&*&#lJqNR+6lz#*gZiwU=@9)zo9O_9O3m`}l$_C@1lPm91^d z%CjdR?809xXoI;dmf~h)T1ZfPvakTP>6GvFvNGPBG|F^m>OocS?^uZxC3{_RF1M_R zMcpMGZGbYN=b{cEc72g>H8z1ttYZ{JLw%~h$Gf86Y@3Y70ZuMlOKhwa8L$>cAVh^y z^cd1l3!0moCyytLKz)(H{l2!KAOVuFxZ$a9{Cfx@znn$Ciz}c#V4~;xz@Z+JyIv`) z9xzMzApCVLI2rrsg}{zhn;r#UNZBLet{v%A8!>Th;!<}hqjP&OoFDOXNtYlF&0Xx1 z%;^sNwxRP;3IS9kb9*&ysFs6p1^b5n9)YH? z-!ZDUa4euRa<`ZAWpXug``eb+hf}~IykFTdG5*zJikWVxK3;ClS*k70q20e$3iGW6 zDOtEs2+fZ4Ut(;Re^YlzcA}6)GwY@4;L*vZ(nd2E`-s9wIvR%V*U!laGtwH8@%>~- z!HV!)S5i)w`Zn$^Wo%o`s^~Iv74YTrRzwN=%epPBbJGEF;R08K3JKeYtUrfD-zfae zRa6=z!73J!ts`HppXPBwTNsi&)BjPn?ScnRilI6nMbmifxYgiw_lg*hMu-HnTJPec zyks;FRiR#{5}fo)bUlLV$mQf;Uss`51N6i~P{{OkdG~jn^By<4@{2_Ot8<5K6Pc-w z)T@Q{`H+dGQChsHjV2b?DvyD4hYd=~y&9%X##<}DBBXQT_8@#VJ5XUgNb_ExKHr!~dY+&0o1iF%kcCB@BTds8q!~5#Pqt8DlG(^YA1R?L zEF#Uj$Z#W1f!k$Hh=|3Um6|nXS8?`MkcA`4MQuef5ohj{k6VKjQIBaB3=7uD908y2 zrOGxz3VA|==_x*2X`O-jhh z9o?7&E?lDIq{M^22{%<%f+Xz=DO5i04pP>kht9xN+G4iQpuHR|- zL$uy4Cw->Lp6@+I?P?wA7#Mq-dwO`^W?OP*l=8S+XCCag+9E;bl)8&53hMZ6v#r|% zNH^bmuw=JBL$t$u(y-ooYaN*(zBn`6$t7H6jhz5`A?zNe8w$1ms7gIwzH-xEz6=fU zixdCh!-|L=ThdgQB2D~;WNGNjH(aqYdnFWVw}xE8ZaGjdzAxyI3S`h)x|;Y-MFJ>^ zt@oncuf*Pcd13U_MP}=&y68k;>S;2&`t1Hfx(3x`GvF;slEDE;{OggIk|bgYBfss9 zx!@{=hp4qEJHI}pDrm5OV%{0to#UTNQJw4|juDK*5=kM$Fo;Imr873KfJR3XTFn}| zi=zDXTci$`ma$}-FS%>;Jy4B#ue;OTp`f}MR~+ZhU^G~|zWdskBH-74L4W*OB1!!F zNjPN3dUm~m*ukx(%iEUl5rW|tlj%4-K;H20>N_;se8^92qYfu{35BBhZi5=0a0NJ% z0kr4k5th##@b@M9!fnK_C*>T0ZqI;+o-f}=W#xzG)vNB~o}v}^%l2dE*V9tz5C5;3 zxhT!ASHvZlM?U}eXSq39ciXuU^~!jYypxpfvLxWOU>^71&SDr2zmr~Am@;Kh1Kia{ z`Ii#aX^5#2e^y{}qKctUM6vvKzesYH;b01GX}))jB()t;|Jupy#cBg`wTb7JfuMhX z3MGN`^DH0zbe3SS2e)JO+o7_*nqITz+hpoQI$MDWr1l!`^ClCbe3H7>evWQX-9ctoxW}K`4s=B95QqrEiGqnA}%C3=GVgYys=Df543v z1_I8iCffkF2Uh)^&yTlEAwNEIbMp;kqN*MQa|H7d9=FSP&wEW%JSsW*QenpizNo&A z+9yXf!LqlOFWxr~9@%{n0GV5z-W1`UySAqt(VYv9|Q*L$~_IPx3e+bvd#Ft2)gn;ROEx_zEL^;fvvFWRC}G&H-{rOFEF!jtXF$1fv%*BjotM37Mc81RP5GKPs=h;4B&#M*`D>Q>oh8&gxA%Z3aXZew}1S^p9l&D5>uxP9OHzX`naWCEU9F@okd!j`!{sPimn{g`ll-F;C-YVG={IAMXtID*Lzg!uBM5rP!; ztZGwI#76N&W$+5l`n15ZkQo&jhH@+0s^byit~?bVbRtLSbiEs=sPI@SyI@MU-qmmk z@`x}#&pl;jRAkrvr?;nn$H&5Ys;c>MrR||#NipN63(6n9Zf*~Xfi-`NWEg%M%OnT& zgEPIgDzmt5BAV#bxecLV^}eu8DNbQv9kH9;n@wx~{N@EUWK}#jzgD)ct9##BE6DFQ z)_jG~QL&eO`yRJ*Q#i;%5O8YYH@F^}l(F zA3*7^&+d;rWdL&vU^HUtsn?0WQj>WEpu%gxme&TPpZ;;5r2*u*=FrKw*zO7JeZQ5K zSundlne+Hh3Bnr3{;m$^d~V-*TWR=NaeNa3hwiM$v~nL7_8QO4-MVQ4+qYWI-nO5d zH*h#jf_FxNgSdC+%_@i`#vdrD%m086uHqhp{dF8^xy+#ZUf=uF5r)iI*wK*|6%~bV zYh&Y|rKvXz5&xM~&=BQWF-by)+T9^Lv%uysOSC#vqgcuNbm1~O`qSkhH65fKfSCV# z@W5`dZ|3l^zZH|i5~?C$jKk(ND`e0@S1e*1sKFY-k^Sm8@N@GZc*y)S4e5}dn7q=@j6x)n>&es92|X@!SFk*my0ZUqF4lncRr z9^a=C@OQHxk#Y_nXOr*C?FS)Y4)aV7|C2Hyq5bW3enlpKs~s8$E)1vV#2-GADWt2r zn&K$2WpjAhTc2gd)QQ#iZ8kbUR~MG0it!2TIGzXrwo|P(H7}Q5%qOh>bkI%=k~n6Z?XtZzvvS z+{AH5^mKZ8+;!T?7YZs{<_tSeDHy?niF56YeXDn zGrpy;jS4kS^!290mOI)Q`9K_nmb zjypZMJAVEF>PvwYD^eQY8TeeU|GIUyII@;ZWkAeFz~CJOmR$l>9+#zHI{CWZ9xvb3 zDR8V_d-dccC80o6gZa5DcLL=HkRAydVlvx@uLaAhn6TD-o=zxmK}sfQw(IT4R*xfq z>h5UrEI)1~y)SXtg3Dn&##~!3U;xSYVf(G^=#2hlAG>!j!|7m*ktRyIvUmYS!O8~t zau2v-Skms!?AZN?*84eB*mb-AbvrIUBg!$@R$rJyp#uC4JpJZIAuCn$;G?sq)CeVA zy`bb&%(2?(f*|}l%?$_rdu)c@Y1>W?I7AwLTxk&QOR4 zllM?!hy-uh;!J&uerxuAuAgrqG}m``yY&YO>Uylz@9tU?j=v1GED0C&7)@vM`d;_J zOkrYTy4Y>R3B63@Ia%!}F}!lNTprt_g+*Hbw$tkjs$}9P{!1$vyJ7+&RYiTX`Qqhe zWq(-^o80k448Jm!Q4!o`*Hxm^paYksT57cV3G?0%O{I`W4xL%eMgb+td7%G5Q#>{? zb39@A$tLV`Pug?$PzwShfMQ_b2Hr<-xKja6*hM1C23PGt_|7}LezhVjEe)|yF1@fD zx%Wg!|}HkbpjPoKg2uK5|VqUy`@<3ZA1Q{T<{HwiIx2Y3#*CUEE66-H*6 zfsyFJ!OpIzvXTAI!&i5d%TN?1m>o#9ebshzz#QZ5cD?o1D{E;fofnwi2~l8E_=WH)0p(-K*4mJ&7Q~a*coZ~b$hIp#(RCMRy_C=9A!9Df^`1;*}EfY z`$5`tYJM_`s^>Aln9)N}dS3izgTrOzj~_qc!Md*d>8*p_ z3!T;|(`VPe_i~`YelIi=hS0uW{3B0UzTMf`*xc?F4fx_I-E6X6&}#I4;3sN*8~z=< z@p2m96B@_E$l0usaUw)ZtR{DcLO5uf32S{1D0mZwW5pzuh&Ma+pE{o?DzA?;ckJzZ zv78w=zatAVrRYE`%^kjaa@nwNy{UHeiw5aMc#!~rP5sbCsss7zZHfIIO72ijC&)Zwp5TF@11yimkFuc$Pm%-o+Raq(zL=XUK)n@u;Bf#2A(f!H}8Ic2|Z-Se}PnXrVn`mPmr}A4_+xP zsvVBYwadFV-qsEddyaGWP0JubUcJc>Je_pBlthr(vu}WUgdD8wP@Qu{SILg^l z`@T(5HHN#w&ENsb-N=qmOg?#A41rWEU+B4z4?f$1GDD5h!(_mftmx?v!p@G4=ZS@0 z(F)j5uXfNLficj1!uf0J$>;726QrG>cX-{b*7IL#ZtuCMk}WnL^6S!4qiMoeTv}=L zU)y+dey`agK57=Y|EmlfaEonhXb_0b`+QI?<8Ru@EGTl@JTIDUDJW}8sSxLkvC*Nj{Xnll|HJ#Zgn-B7<7=-&fa#ODjDIKyVoHR;H5po!}=5!D=?x9$wG z2Y8}YL++Z4H=XL;t!>n?y~}W@MHw_Rib^K;^OgdFxDB2KbL^2Y(o;TPMck$!?vAEN z@sQZP=9_Y7zT#_r`yfj^yL;Z)*xpg#NxycTKLQ?m_=TcukluJLEn3VQWw~NGz~M+! zXMI#vqQDD0?X~LkCHunkYmvISRPNWA3=o!WrBRDzuK*+zfg*H(d7lnEr}h8S0<>IB zX5W3Vc4jHU8eBc8-@N(_ps-PNuCcQp3tstbKY8Uz>(HuojQ>eDKtO2}5b~b)($w<< zE?Io}u(8e0yb8RF!)Jy_j_h`Or0-?ZvX4B?z;2Nt*+vTc&XPIEv)~j3k%(G zI+3|?oW9{R5@EXR@Vi{>y!6@smvo$W`f$Hr%o3&r!O8;_Yy}(0%ke(|MM1j00tXKk zVC1MVfz5329wc~kP6$CuOA8)*auFs^zXS%o0UpnPcPX3zgfkp1DMei0Y*0by#CQea zuNVUaN};MUDCanQXfK|B`YE(E)<7=-mM|+OOuZOGM@;~O22wmt`!ML7a0<$RP3u?U zz~0U1o1C($ysYe&5hF%m=T6NN6c!yc_DxC8F`CT}e!n&U(ixXtg_yWRL|7v=v;pI2 zX=y@pb2E+|IgHlEDy;Zs1*{R_KM11qyrQTF0XY7@fpj~@4xfl0fBYPMQ&OL$k`4q6 z@$vESc)e(AYXji~M+^6(tZ@Io+uK`L{~lv1EG)pyx7{lzB_<6&R&=QB?N^uR?|JM6 zFqMHZ6$%)@fZ2bUgL${!jZItDAv81;l+teDkiY23;c~h#ZQ681^-91^^X~x%K~r-R zz*S!_#V>In94?m=`G*c;(be<2_LKyLX@nA}Oob$>c=OdISo7VNP-HJSQ<2m^8<)+w z5thh!xK%);EI<=V&LGMo2o!rZuf^8YU&CsS$SW@^n+YHwa~Bnx@Ylh^M=iPZ@~cA= zlX??Dr`MY1s00RI3jycwJ3U$Uf~X#pm6c+}mtRcZvTo&t9UUEx*x1g(%2HW>79 zyFLFu#Jb~-JMrU>pQCq5<|CuenLL1V|BWO_N=inE*$lV64J*I=7-oa?baKk?2l1IY zbqcCZRh^2qM%=i4(|T-Ow@RCkf&fYdD2j}nfkQEHKC2 zq)8JYSt4-%ql=($0uBM~c02av?dx%)9~_$b|56YvdS(%#dL+&)@h34OeAv zZ^bLmKZ74vehn^rq0@(8=;(>KV(#q-jfw*YheWxrG{QTLoA^^=T4+nDfb!x)_~F}6 zU=W4!kvYSr7>zmrfH}mHI(6pd2OfX+<+mSr{5eaXj9elZLqS9%R3d@-#_SITR8>0_ zMIE3IL;sv%cxcg!_}7PDkV#Xgn{9UIN0HXZ{qen$GS0?){6q4iCihK?xa9IVCoa0` z#!#=_f!^cj-`13D>pbP1QS_VXeXb}9;KUoBD0FL&0yuI`paM@emr*Gf1 z{dsvd07%cy8Z+U%3%AX=em-@}auD*KoSBD+_t4@D4wV5$9mSsQ8}ZWfi{P}iAhesZJ z0&{O(fXUNmK_z0>xd#p%C8#Ja#FLLdh~}C~fN_Ladtvr9H=s{S8mMRh2L%V~VjP43 zjCI}cQ%LaGn(*e*r{Hk3wGGN1n6YEm&UygI%^lqTs=2oo4jwm2r-JxEe%zM@CTG@e zJOlceK>#5flrl8dSK-;m9z@yE0z`&c?yjwEe09o{DZfh!`43|tW5{iA>MG2+>Pn0nITCSk zS_st9(SbdC_TuADK1E|w6RyAcb_^MF9zcaa!hrcZYzall;e0H5@E)|Z)IpFa;^X^Z z<`p+0zE27`^Hui!8*<-lpgvh=#yB`s5NNH(URMV`d*?OSo9gRh;(L$C&);{FbB@@! zxXd{>-%&VY=9Pl1svsf*hXxMQSc3?tC}Rws1b0>#Jcf zhhpq`Q!#q{BoKWlNKXZm9;MqDxGI6di}&7m4ox+u+WYkFo1TC0U_jDt&W*PoxM|*f zgH^_%Fb;ztK!pM(P>3=?o7aiTk|LBHEr8ADL|kGL1`HU02x}B1qg5;Va;5<$g1lrf z?*$=%>}bcs3-89U;(drThum)oHGSZ0x3`^YX_Nh9nKWrKHf~)1=Q5Dt!-r$vzI`xT z!oPmvxmRXq4;u}U5Um*J^UJc^g#|ZU0ar)Utqo0$Zx0(X40#9g&idFdoHog{ea|7= z%Wr*5d#Cn?s;U8WEB~3gMVBYggMtc!Y@^@cG?TL!^#JGgP z#l^*yBS((Jo;`aI5gGH;vrGT^cw+xtNR(7>|KdVl53d!VRlBuCf&zqT zdmBMhkR<>_5OC@f(?O>fQ56|4KJg&7Z~O@{Q8Bpkrg?}=P6y#K2&bAx2UKwWpS>Jc zIKKY$O_UtUYl)AI9Z_0QS)QAli-LjzgolO4reV@`Jcu>24)Yy zp~HoUh>YF-5k50zb zH{62AUVXqt2{gDh_h;m?2<0H0LREm3AH9acf&-@p=H_PZ-o4u%{K}Z9s4IpI8}XUJ z5{(1fe}Y{wLWt^%=wTC(F>wmQ5;FjcS@R~m8~YIfn6Dzjjuv3mV>ck#OmP2;Z^3Md zhQwvS*N^X43iFqC1g_$lC-23s4eJqVjfSiLxp0l1gt(XP-kCdU)VMX<*Z*n;a@Ol0 z)%Deg43C%^8<#M)U;m824bj*xHPVB!qla-Q{~-GG>GRh=T1t9<{viy=Nk3X!TWhPY zuhZZce&aY3?Pqu0@g9iZc+h#5_XYhf?!!lMA3%|UpuD&UAAk4;!mRPQao(MXOY8$G zP!J!#SLYuZIO{q-0?PD&;>MRBy^hL?l7rKxPR;Tg2j`r|r=;B%)hGQcDLVeG#<)y$ z3^^a!udc?0AMeJjcYeh9JO2i2)(B9GMe|ra6y(7H$|)!aq^yCs@V%d4i_63l^X8(h z;kd@N#WhTeUriyH{_)0J?}Q=50-M7P`Ko(CW7FWGHbJKHpQ}*V)YJl($9wOTnHNLS zn}WD!H7K`0foeWhU~>qI$3F1frPII6kI1-p2~wtAgXGi!Fos4#r6OD^ z;HS2vKZ7ab07S-MU@{CA3&uUY44#DkSn|+9xE*$FpeeV z9t)Iu-EKE1AsX*hPZG^NNh@>oOdQu;Nb`)+2O3~fuamHK!)jEN9>dI8SEFxQ7B~e2 zK?gzwgqVy74-11eJOU9B5r_y6hb1%=27>`qfK~?q0h3jHy8Jbi9x2*8f9^H8$Bv$K zUVQPzc>n$P0RT?s&32hNRy{l)yB@tBLiu5+0w8h~Dm3D1BV#Zy1|8=RDL-8Xg>Z1n zL8+#==j0NgijdS0%zXMCl-r$nZSjkcxdMp`AS~#DeGD&*6a3}sx$puAD>;I&+THLl zGgGMPSADK$9gDSN*Ur~-Mvv3vyBPOnEV*x8<8V3cs5*5VW`l0cqD71FM?Lg99dkMy zTJ}m&0D=C8iQTPOcAC0vHv+gbNG@oim<+a#HhlWQ+nDi}t1)oYco;09h>Enrpf^CL z({ml7OgI5pc>HjKrk)G>tm@xFB(Wcma`cI3)tF z02l;~U8eGI9-?9+v2Xp?IDF9s7&2(QCiDuS^B$-agsMnR&qhjm4(e);L$_};yaosc zJ^ynU$R(Ftf`fVc!cNuIUOeI4X;4`3q|hQ`i4q(rDuC*BBYSY}#+ur?KdgbcT&@tk zLGKG!g4jh)s~Gd1y~1hyyFa6k5cD$!0!fmv>YINfzIR_tn|U!}Vv}Ju8?_ZZSWSm_ zw#)i?ya=TbAVT0OK6>XBR36Lwq`jl#nsq;I1As2Zp=H;&3I@2D8OWRjZ`K4fHC4j4 zhC|NL=!xh z)pk7k-~ya1IeKqLTkFdJ(AwJCb0|qv749~`lBDU}OA@$fg?r4^063ft$5DQ0KPpy# zijeqZjGVIoy)Rn;Q3O<4U8~XFp~8Ls)nI@72BcRCIFW$VBor*V z7KM9uV8DoBr|n$?fkIUk3?4ED-+%E11Wy}udOZ^%@jr!u0D#@*oNW$|MSOfL2=T?h z2wL0)9689z>+BCBsy-Hq;__$O!aJ$%4We02Q2z8lhYd`X-bR z2=IU)1TFSPEPeVR)^NP+=8m@J_j9rbpy=?KSzJ{J9#cm%Tv1+dNv91&Zv-56K+*%| zUSMOd0P)&kKXnWn{{228zFm%9gT^6#@ECMdmZE&q*YG&Juniaia@AAdgC_&V5J2v9 zb}Z{cIIYpxym|#PM+|~Qgw9?R$~ANhp@@p>1qKeU${?w{UKI8C(-?@y2*goJpAq69@&ap1rKB=zo%=w7{W8p+!knA`mmiBbWZwr$6x^C!b2s}Mxe z^FF8B66}H=%-)E_kKXM*UVil4#-^4%zXSkCfi{=douH-RDIjb03SWz%hU;>8!4(la zE(b^(g;s|H?e_R=F@vAw_+ya5t72(TEz-M)*F6@ZJK*D4volLq2gNfA`90;xO0 zrSCkxR{}yq!{O{`M2Mu{w{G3qKZk*2Wo7j%EiFsR$jkx+3978Ey8bKIVBQ>xszOy2 zFvfI$MB^ATY?!sUsOZ6KZ@d`_7d0XTMb=hR=EjQ8{*8wAc1)f+6&GK68FU6CR2IZ^ zqN<3EiGf#^yM|QVL$Nb3x~rTa!WxChs91QphyYLTnNrHW#mC*;h{qnht<~O8m)+RZ zQVaYtf(3&?*KV_QXaYxmgWz0CO8A?+RF^ihL?hsFfvIk=(Ef-TWki!D5=b7cc>qNL z1kqPD6i>56bNYC+t=+7D1mbdMTYU$e!gcV%{ybp40yfXES*k= z*xvoXmXKe^Xfgr+a1>ppW?ES+vcjaVc7>^bd>>W0463NOM00;o16rzq|$EHD4Z$2#!K8xLBPNT0 z>VwHp2Cxpa%0^IP)QpD$@Cv@&dk?i91}KSXaJJiEZ)?|TA>{O;svt?|6`ugkIpX4D z2H$kkE&ux-WYwxwh=_<7r`PKdpVY_4RMY7_0v&ClgwW7Xcw`yXHFf;}oOONe`1p9# z)YKp&BR%B!shXoki#2iXO}D{IjavHDWyjLjmOvL0iiaM46dWQHRqdin?uVuGAfDka ztf4;6uTGCbP7}82oFfE;2)^_<)85AUX<$DN5u7ko9y`R|TKen@1qb&&vUkt+|N8pf z9&elG=zShe)xeRRTxAR(44|s^mHVa{0~l0*(N6t_;3kpo9#3E(mQXN}p|QCUiOIQjWZn?aU`S3fwp$r z9|C4lQ&WQ}Q>K_IDk}FYs+xHH{DtV1{7`Jly)<0pD{5NTsS@9(BoyQD!d*SxXj>oI&fMlF3{XXV6|GIDk>ZfyXbT} z$)CbNyk4)-Xf(lSH2RqNg5q+n6~MeI$H+0`5o)o(X17gFPw#IyTjR~h&|0Dsl9DsG zZQEKc5TwtVeL2pbIs+Uc+H0%u=)DUt>yp_Rd)|crB#mB25(D%={^~>EznpY-obsSh zqo)*c@ZPLDsL5|LjHBk{Ni7_q6rw0=qTw~eYQivvBuZy(X9ghQeWboj@ zIC${j?=s$g{raJ-tPDqw9z}Gdbxvb#%_l_CW9sx7xbgN!Kn0HW=33l6|5jXf`Q^Cq zqRZfBT8Q9}rGpED{=ETV+M>WXgFv-<1Sjn7*k(rSRnwvX4^KZq)#m4|(=+L3qzkmN}_u{5GSK{4IKP&5#nmoR-`sC-1 zpZ?WHi0==z-)JDVHvl8xZ0+2%b6?V+vzNCMg2Q}&Y4CM&Un2|whzN4xI0Qk5$cSEG z{-&^CZ{z!zhK5E6f(WD0$hs4F{}0zeh|%kjPhS}VKjshsco|2|;Nh5i!(XxNpRb{z zq4C@7%#4Y zu7k`3aJaBy#WyH9atyD%`!S$5X~YeFI7g2)KMFmisI57PO&iz2Wq0EGzupd2kb175 z`zAt7bu~;TBdn28;EZ*N@fp*+(BZ;ESoGkXNa&q_OYgZ3LxzonF(edJ9|EQ_NQA&E z%dp#RsH;7N9h=vaBZm$omKGQOrL3&{FR`()dn8Jq+P{DQY5+L*oblMaWm{n6?|%9j z&YLs=8#Zp;%Q;VpkBxuDUR!dVqpBpFC1!$Ti~~ZVz!lzEBjGAQ1b|WyoesR+p&5pN z(;@tn0L%+C)FQHM59H(dFvZ5g6kSSRQuyVD*-#ij2!kSf@cDboFlgv-gh%!At?=mD zQONxqQv?hgA_ewrUB{YhkALTJv|V!j&36M->Rca!3xq-BZ;sYRw4AcRI_exyz}GL) z3F|ywau`qDKMzx9T#B1-yAxh6LXt!%iWkJ|1c3lI*D_}zp%$c=Gmw&&0T6l=?PWYq=W*1k~tjRo(Wsu=n;svu{b1_<5+xT6W#j-XcJZzl*HU0bd|F6fBoSYmupEqxw7#SJ4z+y7& zZER@B=S+>zhlb*^Yvy9vzdlAAxI`0(vl z>ppnrt@O8-{p;=i83O^ysA-HN$dd*)Rr&twbh@A@^67guLZA}}mcRWLvIdX9jkn$o zDo79n0;m)M(Lvyvck=gDaKBmkK%vR~GH}nMPh;u3pJD1H*TCs;W$xO!y&^gy^!>4; z#_9l#H=z4byK5=duU##lJaOXQ3Fn+^O^S)R-QsSll=gfMam_!0V>>`ywGfO(K&0T3 z4n!bYvk22_K(y%;#BX1Qc<%@ppEwVu6B|lL<__4zmsiw%)ai7yIe5aTb1#6^YQ-6~J^XqW1fYV1b0(h8^~Ml#qM`!r zZLO&-ZOu1YEMeoUR;yEGs$=BH5zdP8ieDs!FnY{rz1!`MXG|S#v4q{aXV0EB9*^fj zRb>ew;nA2hZ5HM){2L};cnOFob!`yj49ouU66~$bxM}`E=*{823diZyZ$b#T*NInN zc*e8h%TI@yLQK`Y({ndmI`_^{msf@)3LxCq%idjz*+&3TcB~jKhYfwxvwbu@Bv5g; zHP_?cAH9dipI-`tIUJ00h=hVO8LHxj+hd2$c3*%9PMs*n`X9fA*&rf3BI4DC`i9Cs$sPnJ-EQ~risD1) z-@m_>zGNzRFehzLxS@66ijWy6v**mkxl^X$o6kPL=AYJ}wW;=em(zK^s&H>do1@hl z8Py~b+F~;4KL`m8tqBVYYxQ`&Iz?8@ZSAcFx7#yORg{T^g$417iV#IsB%@a*q9DN< z6Nv$1rr@$Gu0(8HJSY|6(PAo%I{^f$szUL4QFi1YL`i}qiUFyRV6YVPNt+2o#H#PV z;Ro_|PF59nxyR#PI_})_;#=%)a8*US-HlLl2%?MxlV`@(001BWNkljxruJ18Hd~ zSCp1kd^LUAH2kn;&1nyFcYYzRyWtvq`2NS(x_$5VuUBt4|DoG%ix5n@8*5KiKCSLL zX>mcLh>S~Ch2*wyV zI8@b7l|vxvbRa}PxHSrcM~uPH;Uh6*=rBaYCTa9N2!d?*yVfQG@2n$ zA&3tP0sRe11u&K4R8;)wv80ToWIU%V6Ld*tV z-9%#@p%mD>_Q#sSLpz5jItEmOGjL8jofM@gD=$TQY95?7h1PHZ6;G&BzHa)s< z-gV^_rSky1w`1$(8R`AgpFVHOg@gKJ4%8-~$^u0{u5wUGhv_qCA+28u7C-$kt*bu% zWpaG{-+x%M=5J?urXRllap3&Y3(f<8Rsb(8eDLAdK3V?mEt1Fk1hF+%he+aDV|2tf z$B!T1)6$|Qhy4I~=gv?ON?XQNRg-(P%(;cm$IB^g&c~ECvl3 zisY0uh(5j8u{&=yiJ7Y&JMuE*K-wJ&Zc-B2}hBcG*yV?C@RXr6+6Cva_Lh zTo802L4#sQOib_<9D{P7M56`rG8K_=2?V0V_AKD{p0`5^WM5)y|=!; z;dKD7cYe1qaF&&og?$J1qi=HGr22;XH%$gJuDf|YOcpCt27oI;H4+Y)aR?$uZDkqu z@7)c9UV_nR{Oq@wt7pYP7A#l*0PT@h>lS-^`#B%Jw+u^$i~y)Ea33_bSaN!0mH?b9 zpd#1u8Ule!^?(uyy^^vJ(y{pPW7$6 zf*@#&jy;e!gZnhZ!6AwQOlC9M+S*ZFU4_WF-kppj#K({h4!6^WBZUW@$IHsz1^`)6 z?@!Ih0TTpJl?94}jB{`zz~S=1s53z2+E|a4mbmf{9oR|;;UVGS9qnzc7Hh1@N4Ty@ zqm7$zEOuwY|#raS;C=G0>(8-YYGa=WLTr(@Y1r6@ZeoH<9ONO$CCQ=K2UZ13>`5}3ooHj{;;6DHzx!t zW13bxQvgVuQu><^rJYrTY*JG1+le5u^5R2y_qCS+Rn^F7xY}7C;XT1F1Tcsir3@pW zL<7q)$^gbd2!$x=A?OUyhlD^f8X=iXFq%Ul8H@lGeI(N=2w@;Zi}#7fUf^TqXFb3s z{3u+2iUPvIqMw4l`#hS0*!UC}!o#rchwnhRs=-TC29*Ph z5^&Xvn5Y=cyZd3xm_5hjVdM?7!FVVpDr!bmb=~gL{5@HZ+;xkq`gpky zgv@0a%%OOE(Tj-gm4LRk<`sSVruh2x5n*9rg=J-B#pj%J&Y#CXo`3O0>)THD4hJ6j71UKCB<<_OU)$18{&*RG+Ls z@sWe?`+sgS>JF~^;af-|>9S8#L2>*1J8-!8C^m2Y0Zev5A_7#RgP_V-_0?w>IeIMg zArTlic_!xFzX%xvhUT{0UEhb8Ocg_h4Ngf-N%lPYz+G^*)CL)e82GBF3Zmi@@!;dn z!K;XHyKKwj~FpbNKWqkaC39>22e3(#5q%N>-@VQQK{?xv)eNkD8<*G zyn~L`Mu?(>?Ck8T0bt9PEx-CaJ!@&+pVn&x!^Ow#%inu@@y#Bm({d=k0D`E)z#&7m zVq5^!M!GN0UB42X!En0wRFIyl_Q2_I@*ZF!qCnxcwc(#jpT}!Uom z!dU?d&|vt-zFpf;cwlcK=j=q&S zIIw@)0$Xca3joB&#kG}|9J^!q*hz?phy>$W!hsVCLMTQIAC8k{$FcmqchT5z5+_QF z@x#}jB0jko1o?5iYwEtH1pc&pvu54({I#QJ_wP!C+$wHA;ejsWUEuAnJmgjaC&Y zKY9cQ_wKA@jD1EaeKa)GVjeyHG8iPiFVjT2$RHSW6h@-}k#cO`xCYLarbRWiHLmj~ zO+?}0Vw);^|0)^v=HbJ~AP|5M4bVg=MN0n+j2<@ug5C_OGvNFgmtf$K5#XFb1&3tN zqq(UW41n1Zf%E}GFnH7iM8x!h$)Lm0g8c|J8jwABG{A`u#ZG*bpA?`1at35#&G#$e zb~&=r)6-u!8jWgOTiee*>qC`C>9SW}&6PyyDnf-%?GDER6>9j9VWV;91CL?Cv{~SS z0N(wX1w5-ZSrv~Lul@Z#)Ya9(5*~)+-pNhlidHD_n#;HQn7;B-1< zqseq&&8iisuRZ~xJ1)srqtO`*$jr>ds8OR4W(k9=D1q=)k|YR%7(f84j3LAliD6^T z!&NumiHu?6@$m=mp{lYNj45625#cHrQ(y?OVDg1C(a~XxbGzMFR904^ckkX}e0;oU zu~>8o2??g0oSclz%*=5yF)?>Vm_v??&dhCVYHZGTx*SUhC4Kw#%fZ8sFUFILUq{BE zQSdO*ZGr3jT!pC+D4@FSc;<-*v1iY2FwPJXV!nRw-o3jT8XA7HIr@hX@tibi5;kt! zSTpvV(fwXz5l)y`_EJ%JY!IJ#D}`I#&9 z!5+mQR>14T&TSj9am^1X%s&8IM;nYr9ipSGPa6#8f1NmSqN=sE6-I*|q9`h=Doa&W zC(t`B15DLeMF=4vlpwBGFL17kMMXuQ3xWWhqz6+~t&33Yx@9sLDD*l3f{!z|xw*v) z0Edqh1wyyn+(Gx3mK?tDslPphrT<(G!JrRr#`G;7q*H={^8mG|zg8njBD{(mxGy3F zI3ti%im~TR#*w^jSo_`Qxasc4w7QW$y7et2UM}FgsnfCYi;r;P#EIo5lj-^9=4Qe< z6IGRkxZN(HzP?UJsQ^(FAbUK}g;npFCd-t2Kz2r$xkMM12 zIFgf-6;)N+6h(>?FSIKWgjfYXYwM#2~pf}(?YkljuQI%8mkVbIV7oO2|mq@z!IE{+xK zCrN!%;|xX<1RuoTYzc$jXoA&hMR<4^dZ%QdUs^iidnbc25h~N#auudR0P&^8ePe`Q zhf%W*3IbbOGv0jp@7TU!EkaFtL|7w!I&$R5^#FcTGy&cOar-V!;GG7*jQ45kxRwwOmzIh=Krr zAGin=s^UdMeI4p*Pho%FKIHA)jp`G}p>hR^#|cRypq#*HHg_Z>B)pK8miA&@O`Y=Q zYcFYIJ60B0&4&*kuC%nYG!a6g%ZrbIfLZ45vZ$efFz1wj6-J7 z`nVYZXx z5QJbbV`ox=aNZRP6Z9HRI=5^_qY>c|Q7A1bK~-fr`ex*Gaz;>%nu!n&LD1on%df!; zg$Hon^Qo z2hrYK50goc
ZMMXuk0bu&{>G&2jGzl-i{PG>Sxw+5V?T)ML z9c>Hj_Ksx5(T4pyHsIjy%`llQh>VIsbbN0_MMtAouUO~}CWw+0Kncqp51N{q(P8UA zU2PRwo0`zp+6=eT2?~u|8I(XW7+|u5w1=2NzSirFpIa=Jtp^VtbRRo*?C0Pz9Dpo} zA{v{U;ctw2pl&y0mmLy~fus`?kfDnw+W=u(3vufQb|7SjU|9yC2@sb0HiHRRIH@9B9dHpJ=iU)V!{}?2b9ri87>Q<^)Lmf_b`BRYgo}42(t-YHDi01)WB!KqSZ@5GeIk zb{LqffC?0x@j%-;dME;O&cK-#0@~Y~@Xj0mz?QY&gDGxUEMd^;bn}XeikA7`@>?6n zAJ#zf^YbzP_J`=jm&_8w!@?VuFaKZ(fF+|wjh<3lbLxurw$||;Sxu3>9^vGPN>o)G z1M}Tn27pomm2v1qQQM#b&`A;$4hWP(Cy6i_^a=<$85$De2sN8N(py5;73Ci~(J8Hjs^ETjQFG!AEe4JanRZ5t?6}gFsAlYeGPPXI-M?nb2K&9`B2u~(&fG|E<7w8 zAV6(R4JbG$(L5jnpfClLY6%N}{!#1uA>hFY3et(c6DUP{TPwD0*@Ca$dlj{{)i4-L zFo#9d^hr*-V&A^J?KwF)C@L!Y-R9yS%0PZxy+XQc{ykSUwYJV)c;79fz*Vy-3NV{Z zC(I$CyINWrcD?-aTen{Nml?{;nKPr?+S*1nHa1T3csv)_?e=uZ=Q#jC_IhD38o`)G za;&OqL0DK=q0{MXjfsg~9vvN1+0kLMMMp)eE5HBlGcMf2hQ`|ro%#fwo>Q#U3xHFaxsb@lkec{}mIZP#GZj4N>M+#8@bghC~n zcn#+oDFmSeiXuail}>VP-=z09dC9UDPDcj>3P?oiK~s?gdh5d@tdR63xSbsurVJE* zD&TJR6d=lGs>UqOsV1I7q3t)^PA5M7zx8HTyrE{h_oi56W0HY}cl!|b;6sRi8vO5~G8e5xgx_0iR z|B48=?AyC{@B7)=+3y}bdh~k$_dM~$6NdWwda^85N~=g<80`c=6)kkZD3{YU)GH&CSoXx3%Ew<*#AO z`fo9L=3g*r(ljLZO$8?sWQBo)!R2xX0Fqr{W|}288{em35g(} z@OZqas;WXoSuyhW?Ll>ADd}iy9$~XPN95(@Elo&BSQrx(^~qCDJ=NhCm;61C(9qEO z+S*z$RfWst3|cljCrLPyFx069Zi#vm9? z;gO1}jz3&@Y;{kcKYsjpR8&+vKW^N(Zz?J)zHe=5?$=UVOFv)s2EPAfInpw+F?ssM zNX;C8sMt8v)zxTOD?tdPwYV>g5(Npzjvj&6VTZ|NM9!eWVBkHb#r-l5l23G#lK>@N zkEwOy-;AqnbhI|3X#YNZ_3sZ+QFauHtiTXrfYE5E4G*_|HfYG;B`d#OSrZo8iCR7z z#&K2*8G;M;FkA*pvfiS) zd?aT3w)Nh9`}TGD{O#MfW5$dbShZ?Z$z_*ckyUc+*s!*?mKPfv8YeiNHXO*?iTne5 zAc!KYQPD74!jX`egvj_l2oDcOcvu)Xq0s9jFvW{aKYRm5RP;$nLqtS$S2?+EvI|_* zki#J%R)|#a5twkTkA)B)pEUs^;TSZ#Wm2GEKpRXOS+E- zffNa$kaT)P$HwBio9@6xv#!F14J)zY>rc>7bt2rO$ZM^UmNk*q=m$zm%L`LeQ&C=C z{+nNqs%k|+Ns?gD`vi1?n!5mPPck5P7<76gJT52PUJtf!*@*0+W1%oDdn1Y>^m+!T z+XDgt1OgN<#q7UaheJCyhkdYo**{uZn&%}ZCY}}|`u9t9wKeLYL-~6EoHJv_jI5%f zqFFYZZ93t*%CCNeM(>pSKnoiN1Y`3RRXN>Gc>paVjK9cRJf+(79Lk z!r`!?PjV^*Q3yy^+uB=EUtNWY@)8^^D#D=y`*8Aj1tft(lq41!8oEaigl#61Det=L zu3PoQ6HmARp#O~Nf4GFv?@oc zAJNg#OIur8?@mnWizRQq4|8OkuOEX1RrEQS(~Y@T%|LTwHT0q$CUY2;ee@;Fk-Y+3 zx*nCG-R6MD>wB9Zz$kHi_~wh)^ut$(jEWjwQ&W?dp4K17N=tt67?&+uCVlkb@~Jkv z{f5@oj)b=Mjts7PB}I`1N(D)l6{?AMdI1jf?VpX8-~JRjy%C&}9#h5~1Z@q~xb6B` zNb8r$Mvj@l%F8QIT2{&$>*~1OVPi&}s8WJvbEvVJFnTg6A^soJF1YZ(lTR=5;D6-! z-F4E#ETOG#w>SLOg^%H)OD>1%OY;S&oVyb>cUieQH(QI;ndI zv=sE+w_b~*MR^big(yn6^4c3P=az*~If39)pH;O&y0!@x2n0$XyKK1YmN{szIq3F$;6SO;{Q0#$&)8zy$?ClJ%h)rqP+qtQqrA|e!HNC=ZVormdcR_a|K$OgVu#sMIu{iI%^C5+V;8;l++%}ulWpBUHY_X&zCMFiN_?n+n zQ&Z0uRCPcki<5BbBrvyqM2*iiU0wkRQ8U3@d?#J&%yQ|~oxP-VhWhE8Wk&%&bxm-Ux z&WehP)4q4KxL9s#YH~C;HQ5;Bz4z?db(^fJf~Yg$##?U3=tNSp`<$rg7~FdI17Ji1rQHy>-RbSmaCBEl76_m+0=>nG%dcO67hZiE83PBy z%Ng8m_tjNZRiz0D3GYTmM(QU`oCE;K9F+TibuIBp30g6;xR{iemTvWWWXWue#?%XD zf@xGDT^phNdIp>#HKm_MJ4!VMk-C~{y#4Bn5TJ0CDW?&*2evbeqkq;QEO_K;5XnI6 z>l*hYB_&9AZU}TNQon{< zcZ4kf!WdS6_ccsL{UZSIppUnZXY$1>b{_k#$Jh=igjXsEBh z(d+d#u3WY9n~@VHBoyas>EM|&Xa3@Am~qiXsHv(308AP)W}VYv&oo=YFl^L#q^70& zA`jYw&?W#vk&xIMf*?T<1n6`+NRohETQ?$q-%cWlQqQrkFEhlb7w1l$fk&TQf>5&s z_4W1dTdk1?hmIVb=6~r!k38JtxM$6p4FD)8$Oi!Q>C@-dYp%QQh}+{{uBr;ctZAavCIr_$uWbz_H@PXlbZ+g&0FV!Jqi}T|%t2t2YXteg4rJy~#qZ zpMNhbmPifROrU|aP5`z7^cjEacALQSB{Lb#xtG>wJx_(`0-KkSHgF)9T%wi#e zP6t)xKIpclL*Wn8NWkO?1W|{KoIw~h{#+0W)YsL(Ve8P_9rnzohPs8J=Fl-#tF;|K zOKokf^OsylOr$l^?eScos_JH!+c`j|H=uvcAS`_FNl=~M2f=_3iP|~S2{;%Btkwvu zUbzAuw-Xr|8BkS~+Z-LFthfl{CQX4(Z`4410_j@mCIp;mX}yHxRAgmlp!mp9G&TO8 z_O3KMiYi^d=T!CHo$l-_WG8_@*i1k{Kz0;&MBE1CjvJ2qHg|Bt8P^fT z;E2jjBrHilAS95D?0Y&(cPHsicURT9KdL(kf#B$jjGBIb01r=9SJkQd&iT%_ystT| z#a#b@(Wpx|8uSfK_4Urgq-0uIS=qe_&dA81Tef@=;C8t$G3X7~18keMt!=(r5QEej z9Y&{(!ylephSao)@M_b0 zGquxZ&V;?wg~JCkFlywOu;|2L&(58H+r5Kg5{V=VAe4HdTK#2i2?Rlef@7$uD#z#R zSK~nDcW~Nm9tkly6RgwevJ3`8nMR}eYiw+6v&m#~sMTsTG&HDBoH(HaaJ$3dnBuT^ zOoS*B!YiTDXplT|9F{)&0@ONxFnV@H>RINKjx_KkoJ8U452uc{EN)2I* zRqAxQTDRM+7DX|}>2&%-5(q~q6lx7bMi4W2Fc#c-FVZH@0tf|RQa6`ZZ;s^N&qE1= zC;^?8W-M7azw_R^ZZo~`=T`-vf&82B3(#NZ=yYzm|KX*WJ8uyL=I$PNiAkV@pxxSr z2No@WnwREQR91c+JvauHRq|%;)?04DCmS~c0OnjbKlti-3+|2>6!(N$tv1R%5B#|= zl*yvflmTv00;g0!5CMx8bmBlOCx8c~78FD7rsRN+~2s{%Ax| z1fdi>r2r^~j~S0Eue%XxmrjC8sfM=zN$S0FupX{m!WcwGp;r+soOcx(>M9Ws5b#lL zZS5UHk`ixisHy%~gbEKWTaGKPm@1Rjov*Y5o^ZD}^GyjVN{g{?*LIvfegsANxp29i z9!5tFk|cV#wYMv2D1h6*OptGY5V=ldHPeo;Aw{ZaI zHVyztPDwQ{x&4M0De&??c4u$b7z1XAkYsN}WZi;1-X3#8JpJL41fBrlQ88F}&jaYZ z<6cx(lp}NRx2P&DMsrgm+AJ-wJLEEWCP}jT6;(i?R3R!l7K39GF>%IKNKQ@$rM#zd zm38~(eZ&LLRZU|IDy>>3v12R(0E&xBJ{dVOwXUF`aO(%JEH%arUXSpY1X=M{k{g}~ z>lrZF{gDd;MFtFxO~Rc^k|DafaHhBr_4U=LE-weqBsc{IjaHAy$Vem)8xDOyFw`2Y zeBZ2xWwIxHes0c#Fj>SyA{-QyN9NvbC^?n;jJc&{AAsMqaa_C=#AIAe818=YwRey> zdIA^|y8UbFiTyX%Rba`Y`G}1kG$J%KEWdB;zBk_Av<-JHmvJ5KD9QUX3b_TC63=k6e#Jn$SG}_ooMiWq zdR)B1yc1aV%#*t#!vbdJojA@g0B}*FSZcLeAxV`g}~3xA9Z#0 z_w3rW3+ZD=_xpQyEtu<_P1%!6em`!_@mz30|jv5tPS5{E?>=XCG(q@*Mt~|4S|1%te zMn$3E7zAf0$_u`SwY46cf_hd@2ErsybMpD8M$#+!*Ow)R5dz9FlosaVjhB}lOC1(_ z)j%`70GkKTA9zo2D!@(?sN z|6r`2WgW;^*Jf>-G$c7GR;@R6_f1=DUFhtRWr`W7hoO%`lzS4Pgif=EmPD22t`fK40LLKIvG4GleDHk)?wV&kTdQJk|6CW8^CzyR=^9Lx|U296RCN$QcJ_EZj&UQ|imCzNSFf$Yrvc>YgM zlt_{=zO~uX)zM*v&p`fz@Hd(K-7e=MjlqNolV`{sZQY$|w3}x(HaP{24fQyFBx~`Y zsPL`Dr4{BM`t4EalZpE|Mwedr=)Hfm|IuZWQlSZW>FXup3Zca|6wWayG_@SfF z#kjM}wa@Y`rEc-JE~fycQVEWukRZzbdw(*;=UXu@v=8yl%p$}3_uocST_uc0lMoph z`9e}slC83`@|=4!o10iidq-hNaL}7Ozun#?xg3`(!O?$y_9-@fuo~O5k3f~2hSa;B z!|26-L~z;^NTwhtDF;vnC{zda_x29w?e{$VN6ejn+u4`) z@&Sl0NSqoHfUh>L#V70D0|na`8WMC}VeuJTbW98?FHpx57aNbV@=|E^+VN5GDf<@Q zzf>I@9@`HsmaM-adD`Gi8V*K-3WT^p!1ZwRl799W=iA)&VLI^JW-^BtsD8qDD>$FhPPs zpb$AZEXhQ`Ng0%kKu`m&x*8l`^)#w>{R5qL7sgMTh8yqrJrWWUp;D>4s~RN1h0-&{ z*zmWv?D@xX-s^I7K7j#_i)J8W$Bs1>78ce8hK6b0dVdqNx&TN-@FeWcv$0^n>D(h& z_Sk)pB%$J##kVEB|MuTpKXjcDBT|r;cM7p_anUxrc=Ym_*9KiXe-WHS2`VrMLB5SOHn!{Lkr_-^<1(l&GDBX+xEzqzT=3ILBk`Vd}!S*u4xbYJp5_Zl~@BKFcLquc*BBCO3X#ajwVczK(aq)5MX3w6@F4R`r z)MUmr*Ia|_tSsy3#KG$i?cY7=)T!L4h=@oSLc+j#^9a4-{-l@NrBN$-%I^ERQ+@T^ zfnKD_eT;|@aA4phH&(s192tAQhFYb9zbW9ulldptj2blx4Gj(bK8HiX?k;`IaKE4+ z5;SfSB%J|7Vqg-%d0NVUIN`+rl`#n66K5lK`nAyN15k73G!A~X8Q*UG3{}Oa@V6Cz z9{lDzEA#E0o!zA>hYn{C=oRFm7)VJ;2}DsuNqPBaE8lthp`z3I>VV(~Bo0gICJQ)= zT-s9(E-`}WxI`Sv*ae%l&7@YVU(d+M_(5N|sHh09Jo_ZR+5KH-o2~r=N<_`>ZJ$qU zsV~Ejq$KbflT6Iq{ZUg->P>=Htv);W&H9RR_GOqS=j=heMVO-7fsfa&!RC+FK&2#z zjEpQTuPC4P$it6d>(>7DXNC<)(3;Jy&$-^*sWG1^LCuP#2p<#)Wmp^}o_IKUf1HId zPR@h#j6mxbil7OTk#gli1P>m9>dJDoRF`4nrcDp=I$djGOiW&LON$5qH(q@s3W^H` ztbtrK7b`tI9XUBU7&|(3;;GX`nY`ABho4%Csnf59TW|wRl3PCs@rdUzFmMn=H=cQ5 zA*#zu>q)7ut zsYhiBJw=24xK)xT94t{m&NoO(P*6D9Tkzs@|BKus2cg!gpf~8sLqbBv96xr#@Ur|Vt4P9I!ST$plj=RcK=RV7d;co_V2P$;xeC{!@0)pC)bm-Zih zH|sqsUhkqp0u3nT;dXam_l~Vt{^GMJIDH%nC4s@3gNuZ2ea4IB>bo-26 z?qZix1%31oB+s0W=qcC3ZnNpERmC$X6Cc!TRUQ6*`l{v@OQ+93er9BR_Z{ZUnS;Yw zStsM;` z0@2=%qX#qa+#espo^Q9JtKAB%R*m40pq0DduAckpo|Vq>@_$XjJ18_1EwwcdxW?b4 zM|g}UZ$ls^PJl3BF09)=LfHqeL1?T;L~;t08V%eMg;vg%{OEB}Lhh-Ncpj?2V8lxZbL%;8yCG|>QwC8yBEoc@srza*3TNut-(5D z0Fs7}#PH#%2nh*6dwV-ho;Z%21A7o065aQ-E>=Xlp|5+E;O8 z^zT%{ZBK}DcH9+0{}2-(xf>R6%}(^T3Y70T&@tWTmr}Q z=yHkB8%+oZ2msG3pi-$}v)N#4x1qhI0ffleXP)Pw(`mpM3IYQI-#T;V%ql_%X3w07 z?K^)wT~}mi@TavmzdRqa@01OscW6QcpaOI*2kiU50{`Wk&@`7K_NGUWJnt3+1V=+8 z0P!Tv-5ek=21S4F7bH2lClQDeV5=|1$<)h27;AwzJfW)C!ITk=fxfve>aJoTr zwm`LhDfHH6gx_#4#x8mSTu=a*NTBqP(YgDnP2~wgJ4b+~avWRz0!p`T1Pcg|E#08@A;dw3f2dqvb_))ZJTdn<;{z5_!S+yiBFER;?Wf~Sj;F+c&p z4S?qXYkdt)t$h_`UvEaI+5|d!8bnPf2y@B72+j>~S}4AL149#HX8ku9EPV#@3&icW z{|+C0wBdrs%$hL`J9h2)S+1|We7@81r%#m(A|QHJ5Q&|e#Oxbjb6McO?HzP&Sq;YQ zNMHOEB5t@F2E7TKD8WS_hoIR0%@5K)-$q zsVT|Rjvqh1*B8a|0Vhx9{@^jCW##|z>)vw1JmeRghO*N-lG6tcL(C{osXoqTOzG2i z0sx>QpwPmRd^y~676U3Bs@A`Z@~s~rC@36U$_S_^Md^;eW9Nf6pz?G9#1XR~B~Av@ z1j$R3OgAsL;eyDnoT~%M@?$U>{a!Mg%`F4v8}O|lzfh#43`0##OSD^bb23KHlvH1W`U&sRXIbgYMMM zjt;xsFp$Qd`aZ#5E=H!N;zaJL#;~ZEWR0aPTX=3JxYf^svv&b1PdJO>{EUpO&ot-; zSkf4{XWjw97$Un*Diy#hJc)H#R8%G`>t$_Ju54y*CpeyC1MMC74CEJ#+> z01X3gE{Bx;B@B|)rc!IK?d;@op34@a2jVG8Z-R&kss>O}xS0JKU_p&s<`7I0ag!9?xr3afKWjg5^* zZoc{E0X5{lDAuoiWyveyMo&vy>#*2<$LR+_{GS)$oOlx?h0@cT+bbTdARrrF0L}gi z;+SjY>Cx^4cnWm60Idq>vV-LA1YdUwyh7Cw8W}dZqO`PlppLI^1^E@j+|Z2Z5s6}J zlVzdFKR8_>wbe==EP-PAUC=g^_K4pQCa(|xm`ETAq8z>Ia>@U%R0A3XXh9|vyWWG^ zT-;(X8E$>>(TAccN=u6;O`ZUsf%x#NLvUD+D0_WlLtEy^UYm5S_-gswl$0Ds^M z5YgdroxF7@6io2czPsfRuEJ6QvV9HI1(|N6mVD^was;)uv~6sxs}liW|91y{2I9l7 z8&j{i0szq1-o7Md^yQ-rj@C-mipB6-IuB?%2`G9+rzimw44jHVS#l8gZWBV(T@`~O zL*u6}dgx9-md*R{!w-D}>%(sgk3Rf706;j^OoM`hpVz1rOdXcU6n8w&$QL$-`&KhP zZagEi7BdY2e5v_Ey^uqdx_Tsn5R zZ?JsuVc`w8@`-6<{Cy+ggAYFV;DZl7_~3&NKKS5+4?g(dgAYFV;DZl7_~66;12yE8 UaBka@*8l(j07*qoM6N<$f^aN~WB>pF literal 0 HcmV?d00001 From cf7b59538572c3cb49c8e54aec3e7a7f0da06b42 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 9 Sep 2015 13:18:52 +0100 Subject: [PATCH 170/337] Improve error messages from oneOf schema errors oneOf schema ValidationError takes a little more work to parse and pull out more detail so we can give a better error message back to the user. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 51 +++++++++++++++++++++++++----------- tests/unit/config_test.py | 5 ++-- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 44763fda3f..971cfe371f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -107,16 +107,6 @@ def process_errors(errors, service_name=None): def _clean_error_message(message): return message.replace("u'", "'") - def _parse_valid_types_from_schema(schema): - """ - Our defined types using $ref in the schema require some extra parsing - retrieve a helpful type for error message display. - """ - if '$ref' in schema: - return schema['$ref'].replace("#/definitions/", "").replace("_", " ") - else: - return str(schema['type']) - def _parse_valid_types_from_validator(validator): """ A validator value can be either an array of valid types or a string of @@ -149,6 +139,39 @@ def process_errors(errors, service_name=None): return msg + def _parse_oneof_validator(error): + """ + oneOf has multiple schemas, so we need to reason about which schema, sub + schema or constraint the validation is failing on. + Inspecting the context value of a ValidationError gives us information about + which sub schema failed and which kind of error it is. + """ + constraint = [context for context in error.context if len(context.path) > 0] + if constraint: + valid_types = _parse_valid_types_from_validator(constraint[0].validator_value) + msg = "contains {}, which is an invalid type, it should be {}".format( + constraint[0].instance, + valid_types + ) + return msg + + uniqueness = [context for context in error.context if context.validator == 'uniqueItems'] + if uniqueness: + msg = "contains non unique items, please remove duplicates from {}".format( + uniqueness[0].instance + ) + return msg + + types = [context.validator_value for context in error.context if context.validator == 'type'] + if len(types) == 1: + valid_types = _parse_valid_types_from_validator(types[0]) + else: + valid_types = _parse_valid_types_from_validator(types) + + msg = "contains an invalid type, it should be {}".format(valid_types) + + return msg + root_msgs = [] invalid_keys = [] required = [] @@ -200,12 +223,10 @@ def process_errors(errors, service_name=None): required.append(_clean_error_message(error.message)) elif error.validator == 'oneOf': config_key = error.path[0] + msg = _parse_oneof_validator(error) - valid_types = [_parse_valid_types_from_schema(schema) for schema in error.schema['oneOf']] - valid_type_msg = " or ".join(valid_types) - - type_errors.append("Service '{}' configuration key '{}' contains an invalid type, valid types are {}".format( - service_name, config_key, valid_type_msg) + type_errors.append("Service '{}' configuration key '{}' {}".format( + service_name, config_key, msg) ) elif error.validator == 'type': msg = _parse_valid_types_from_validator(error.validator_value) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 90d7a6a26d..f55789207d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -183,7 +183,8 @@ class ConfigTest(unittest.TestCase): ) def test_invalid_list_of_strings_format(self): - expected_error_msg = "'command' contains an invalid type, valid types are string or array" + expected_error_msg = "Service 'web' configuration key 'command' contains 1" + expected_error_msg += ", which is an invalid type, it should be a string" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( @@ -222,7 +223,7 @@ class ConfigTest(unittest.TestCase): ) def test_config_extra_hosts_list_of_dicts_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" + expected_error_msg = "key 'extra_hosts' contains {'somehost': '162.242.195.82'}, which is an invalid type, it should be a string" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( From 1007ad0f868e00c61267e7b9eb059b5b811a84d9 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 14 Sep 2015 17:23:05 +0100 Subject: [PATCH 171/337] Refactor to simplify _parse_valid_types Signed-off-by: Mazz Mosley --- compose/config/validation.py | 43 +++++++++++++++--------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 971cfe371f..dc630adf27 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -95,6 +95,12 @@ def get_unsupported_config_msg(service_name, error_key): return msg +def anglicize_validator(validator): + if validator in ["array", "object"]: + return 'an ' + validator + return 'a ' + validator + + def process_errors(errors, service_name=None): """ jsonschema gives us an error tree full of information to explain what has @@ -112,30 +118,20 @@ def process_errors(errors, service_name=None): A validator value can be either an array of valid types or a string of a valid type. Parse the valid types and prefix with the correct article. """ - pre_msg_type_prefix = "a" - last_msg_type_prefix = "a" - types_requiring_an = ["array", "object"] - if isinstance(validator, list): - last_type = validator.pop() - types_from_validator = ", ".join(validator) + if len(validator) >= 2: + first_type = anglicize_validator(validator[0]) + last_type = anglicize_validator(validator[-1]) + types_from_validator = "{}{}".format(first_type, ", ".join(validator[1:-1])) - if validator[0] in types_requiring_an: - pre_msg_type_prefix = "an" - - if last_type in types_requiring_an: - last_msg_type_prefix = "an" - - msg = "{} {} or {} {}".format( - pre_msg_type_prefix, - types_from_validator, - last_msg_type_prefix, - last_type - ) + msg = "{} or {}".format( + types_from_validator, + last_type + ) + else: + msg = "{}".format(anglicize_validator(validator[0])) else: - if validator in types_requiring_an: - pre_msg_type_prefix = "an" - msg = "{} {}".format(pre_msg_type_prefix, validator) + msg = "{}".format(anglicize_validator(validator)) return msg @@ -163,10 +159,7 @@ def process_errors(errors, service_name=None): return msg types = [context.validator_value for context in error.context if context.validator == 'type'] - if len(types) == 1: - valid_types = _parse_valid_types_from_validator(types[0]) - else: - valid_types = _parse_valid_types_from_validator(types) + valid_types = _parse_valid_types_from_validator(types) msg = "contains an invalid type, it should be {}".format(valid_types) From a594a2ccc25206cc7794ccf8db47982eebc34ecb Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 7 Sep 2015 16:45:58 +0100 Subject: [PATCH 172/337] Disallow booleans in environment When users were putting true/false/yes/no in the environment key, the YML parser was converting them into True/False, rather than leaving them as a string. This change will force people to put them in quotes, thus ensuring that the value gets passed through as intended. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 10 +++++++++- docs/yml.md | 6 +++++- tests/unit/config_test.py | 6 +++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 6277b57d69..baf7eb0eec 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -36,7 +36,15 @@ "environment": { "oneOf": [ - {"type": "object"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9_]+$": { + "type": ["string", "number"] + } + }, + "additionalProperties": false + }, {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] }, diff --git a/docs/yml.md b/docs/yml.md index 9c1ffa07a4..17415684db 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -184,17 +184,21 @@ Mount all of the volumes from another service or container. ### environment -Add environment variables. You can use either an array or a dictionary. +Add environment variables. You can use either an array or a dictionary. Any +boolean values; true, false, yes no, need to be enclosed in quotes to ensure +they are not converted to True or False by the YML parser. Environment variables with only a key are resolved to their values on the machine Compose is running on, which can be helpful for secret or host-specific values. environment: RACK_ENV: development + SHOW: 'true' SESSION_SECRET: environment: - RACK_ENV=development + - SHOW=true - SESSION_SECRET ### env_file diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f55789207d..0c1f81baa6 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -270,15 +270,15 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual(service[0]['entrypoint'], entrypoint) - def test_validation_message_for_invalid_type_when_multiple_types_allowed(self): - expected_error_msg = "Service 'web' configuration key 'mem_limit' contains an invalid type, it should be a number or a string" + def test_config_environment_contains_boolean_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'environment' contains an invalid type" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( config.ConfigDetails( {'web': { 'image': 'busybox', - 'mem_limit': ['incorrect'] + 'environment': {'SHOW_STUFF': True} }}, 'working_dir', 'filename.yml' From 8caeffe27eb29b830f181c086302ceb724397571 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 10 Sep 2015 16:25:54 +0100 Subject: [PATCH 173/337] Log a warning when boolean is found in `environment` We're going to warn people that allowing a boolean in the environment is being deprecated, so in a future release we can disallow it. This is to ensure boolean variables are quoted in strings to ensure they don't get mis-parsed by YML. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 3 ++- compose/config/validation.py | 29 +++++++++++++++++++++++++---- tests/unit/config_test.py | 28 +++++++++++++++------------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index baf7eb0eec..66cb2b4146 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -40,7 +40,8 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9_]+$": { - "type": ["string", "number"] + "type": ["string", "number", "boolean"], + "format": "environment" } }, "additionalProperties": false diff --git a/compose/config/validation.py b/compose/config/validation.py index dc630adf27..0258c5d94c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,4 +1,5 @@ import json +import logging import os from functools import wraps @@ -11,6 +12,9 @@ from jsonschema import ValidationError from .errors import ConfigurationError +log = logging.getLogger(__name__) + + DOCKER_CONFIG_HINTS = { 'cpu_share': 'cpu_shares', 'add_host': 'extra_hosts', @@ -44,6 +48,21 @@ def format_ports(instance): return True +@FormatChecker.cls_checks(format="environment") +def format_boolean_in_environment(instance): + """ + Check if there is a boolean in the environment and display a warning. + Always return True here so the validation won't raise an error. + """ + if isinstance(instance, bool): + log.warn( + "Warning: There is a boolean value, {0} in the 'environment' key.\n" + "Environment variables can only be strings.\nPlease add quotes to any boolean values to make them string " + "(eg, '{0}').\nThis warning will become an error in a future release. \r\n".format(instance) + ) + return True + + def validate_service_names(func): @wraps(func) def func_wrapper(config): @@ -259,15 +278,17 @@ def process_errors(errors, service_name=None): def validate_against_fields_schema(config): schema_filename = "fields_schema.json" - return _validate_against_schema(config, schema_filename) + format_checkers = ["ports", "environment"] + return _validate_against_schema(config, schema_filename, format_checkers) def validate_against_service_schema(config, service_name): schema_filename = "service_schema.json" - return _validate_against_schema(config, schema_filename, service_name) + format_checkers = ["ports"] + return _validate_against_schema(config, schema_filename, format_checkers, service_name) -def _validate_against_schema(config, schema_filename, service_name=None): +def _validate_against_schema(config, schema_filename, format_checker=[], service_name=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) schema_file = os.path.join(config_source_dir, schema_filename) @@ -275,7 +296,7 @@ def _validate_against_schema(config, schema_filename, service_name=None): schema = json.load(schema_fh) resolver = RefResolver('file://' + config_source_dir + '/', schema) - validation_output = Draft4Validator(schema, resolver=resolver, format_checker=FormatChecker(["ports"])) + validation_output = Draft4Validator(schema, resolver=resolver, format_checker=FormatChecker(format_checker)) errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] if errors: diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0c1f81baa6..f246d9f665 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -270,20 +270,22 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual(service[0]['entrypoint'], entrypoint) - def test_config_environment_contains_boolean_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'environment' contains an invalid type" - - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - config.ConfigDetails( - {'web': { - 'image': 'busybox', - 'environment': {'SHOW_STUFF': True} - }}, - 'working_dir', - 'filename.yml' - ) + @mock.patch('compose.config.validation.log') + def test_logs_warning_for_boolean_in_environment(self, mock_logging): + expected_warning_msg = "Warning: There is a boolean value, True in the 'environment' key." + config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'environment': {'SHOW_STUFF': True} + }}, + 'working_dir', + 'filename.yml' ) + ) + + self.assertTrue(mock_logging.warn.called) + self.assertTrue(expected_warning_msg in mock_logging.warn.call_args[0][0]) class InterpolationTest(unittest.TestCase): From 4b2fd7699b1905de2b2f03be5d6a6ba442b5653f Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 10 Sep 2015 16:54:31 +0100 Subject: [PATCH 174/337] Relax constraints on key naming for environment One of the use cases is swarm requires at least : character, so going from conservative to relaxed. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 2 +- tests/unit/config_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 66cb2b4146..e79026265c 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -39,7 +39,7 @@ { "type": "object", "patternProperties": { - "^[a-zA-Z0-9_]+$": { + "^[^-]+$": { "type": ["string", "number", "boolean"], "format": "environment" } diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f246d9f665..ff80270e6d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -287,6 +287,21 @@ class ConfigTest(unittest.TestCase): self.assertTrue(mock_logging.warn.called) self.assertTrue(expected_warning_msg in mock_logging.warn.call_args[0][0]) + def test_config_invalid_environment_dict_key_raises_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'environment' contains an invalid type" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + config.ConfigDetails( + {'web': { + 'image': 'busybox', + 'environment': {'---': 'nope'} + }}, + 'working_dir', + 'filename.yml' + ) + ) + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 2f4564961123feb2f033aa91ba1b2b7938b32c62 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 17 Aug 2015 13:15:52 +0100 Subject: [PATCH 175/337] Handle invalid log_driver Now docker-py isn't hardcoding a list of valid log_drivers, we can expect an APIError in response rather than a ValueError if we send an invalid log_driver. Signed-off-by: Mazz Mosley --- tests/integration/service_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index bb30da1a1b..17fd0aaf1b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -864,7 +864,10 @@ class ServiceTest(DockerClientTestCase): def test_log_drive_invalid(self): service = self.create_service('web', log_driver='xxx') - self.assertRaises(APIError, lambda: create_and_start_container(service)) + expected_error_msg = "logger: no log driver named 'xxx' is registered" + + with self.assertRaisesRegexp(APIError, expected_error_msg): + create_and_start_container(service) def test_log_drive_empty_default_jsonfile(self): service = self.create_service('web') From fb96ed113a4757372e01d1403587af73c9e77bea Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 17 Aug 2015 13:17:01 +0100 Subject: [PATCH 176/337] Stop sending json-file by default By doing this we were over-riding any of the daemon's defaults. Instead we can send an empty string which docker-py sends on and the daemon interprets as, 'json-file' as a default if it hasn't got any other daemon level config options. Signed-off-by: Mazz Mosley --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 7406ad80dd..7e035e29f0 100644 --- a/compose/service.py +++ b/compose/service.py @@ -657,7 +657,7 @@ class Service(object): cap_add = options.get('cap_add', None) cap_drop = options.get('cap_drop', None) log_config = LogConfig( - type=options.get('log_driver', 'json-file'), + type=options.get('log_driver', ""), config=options.get('log_opt', None) ) pid = options.get('pid', None) From 39786d4da7127bb1a0898da8c63ad77ec0adf8a3 Mon Sep 17 00:00:00 2001 From: Christophe Labouisse Date: Mon, 14 Sep 2015 15:02:15 +0200 Subject: [PATCH 177/337] Add new --pull option in build. Signed-off-by: Christophe Labouisse --- compose/cli/main.py | 4 ++- compose/project.py | 4 +-- compose/service.py | 4 +-- docs/reference/build.md | 1 + tests/integration/cli_test.py | 46 +++++++++++++++++++++++++++++++---- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 61461ae7be..9b03ea6763 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -153,9 +153,11 @@ class TopLevelCommand(Command): Options: --no-cache Do not use cache when building the image. + --pull Always attempt to pull a newer version of the image. """ no_cache = bool(options.get('--no-cache', False)) - project.build(service_names=options['SERVICE'], no_cache=no_cache) + pull = bool(options.get('--pull', False)) + project.build(service_names=options['SERVICE'], no_cache=no_cache, pull=pull) def help(self, project, options): """ diff --git a/compose/project.py b/compose/project.py index 9a6e98e01d..f34cc0c349 100644 --- a/compose/project.py +++ b/compose/project.py @@ -257,10 +257,10 @@ class Project(object): for service in self.get_services(service_names): service.restart(**options) - def build(self, service_names=None, no_cache=False): + def build(self, service_names=None, no_cache=False, pull=False): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache) + service.build(no_cache, pull) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index 7406ad80dd..d74c310b91 100644 --- a/compose/service.py +++ b/compose/service.py @@ -700,7 +700,7 @@ class Service(object): security_opt=security_opt ) - def build(self, no_cache=False): + def build(self, no_cache=False, pull=False): log.info('Building %s' % self.name) path = self.options['build'] @@ -714,7 +714,7 @@ class Service(object): tag=self.image_name, stream=True, rm=True, - pull=False, + pull=pull, nocache=no_cache, dockerfile=self.options.get('dockerfile', None), ) diff --git a/docs/reference/build.md b/docs/reference/build.md index 77d87def49..c427199fec 100644 --- a/docs/reference/build.md +++ b/docs/reference/build.md @@ -16,6 +16,7 @@ Usage: build [options] [SERVICE...] Options: --no-cache Do not use cache when building the image. +--pull Always attempt to pull a newer version of the image. ``` Services are built once and then tagged as `project_service`, e.g., diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 4a80d33695..9dadd0368d 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -97,6 +97,19 @@ class CLITestCase(DockerClientTestCase): 'Pulling digest (busybox@' 'sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') + @mock.patch('sys.stdout', new_callable=StringIO) + def test_build_plain(self, mock_stdout): + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['build', 'simple'], None) + + mock_stdout.truncate(0) + cache_indicator = 'Using cache' + pull_indicator = 'Status: Image is up to date for busybox:latest' + self.command.dispatch(['build', 'simple'], None) + output = mock_stdout.getvalue() + self.assertIn(cache_indicator, output) + self.assertNotIn(pull_indicator, output) + @mock.patch('sys.stdout', new_callable=StringIO) def test_build_no_cache(self, mock_stdout): self.command.base_dir = 'tests/fixtures/simple-dockerfile' @@ -104,14 +117,37 @@ class CLITestCase(DockerClientTestCase): mock_stdout.truncate(0) cache_indicator = 'Using cache' - self.command.dispatch(['build', 'simple'], None) - output = mock_stdout.getvalue() - self.assertIn(cache_indicator, output) - - mock_stdout.truncate(0) + pull_indicator = 'Status: Image is up to date for busybox:latest' self.command.dispatch(['build', '--no-cache', 'simple'], None) output = mock_stdout.getvalue() self.assertNotIn(cache_indicator, output) + self.assertNotIn(pull_indicator, output) + + @mock.patch('sys.stdout', new_callable=StringIO) + def test_build_pull(self, mock_stdout): + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['build', 'simple'], None) + + mock_stdout.truncate(0) + cache_indicator = 'Using cache' + pull_indicator = 'Status: Image is up to date for busybox:latest' + self.command.dispatch(['build', '--pull', 'simple'], None) + output = mock_stdout.getvalue() + self.assertIn(cache_indicator, output) + self.assertIn(pull_indicator, output) + + @mock.patch('sys.stdout', new_callable=StringIO) + def test_build_no_cache_pull(self, mock_stdout): + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['build', 'simple'], None) + + mock_stdout.truncate(0) + cache_indicator = 'Using cache' + pull_indicator = 'Status: Image is up to date for busybox:latest' + self.command.dispatch(['build', '--no-cache', '--pull', 'simple'], None) + output = mock_stdout.getvalue() + self.assertNotIn(cache_indicator, output) + self.assertIn(pull_indicator, output) def test_up_detached(self): self.command.dispatch(['up', '-d'], None) From 7c32fcbcf58831d51e1cc981e4880ee554809ddd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 10 Sep 2015 17:49:48 -0400 Subject: [PATCH 178/337] Add 1.4.1 release notes and download instructions. Signed-off-by: Daniel Nephin --- CHANGELOG.md | 16 ++++++++++++++++ docs/install.md | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f18ddbf8d..a054a0aef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,22 @@ Change log ========== +1.4.1 (2015-09-10) +------------------ + +The following bugs have been fixed: + +- Some configuration changes (notably changes to `links`, `volumes_from`, and + `net`) were not properly triggering a container recreate as part of + `docker-compose up`. +- `docker-compose up ` was showing logs for all services instead of + just the specified services. +- Containers with custom container names were showing up in logs as + `service_number` instead of their custom container name. +- When scaling a service sometimes containers would be recreated even when + the configuration had not changed. + + 1.4.0 (2015-08-04) ------------------ diff --git a/docs/install.md b/docs/install.md index 371d0a903f..5496db2eed 100644 --- a/docs/install.md +++ b/docs/install.md @@ -52,7 +52,7 @@ To install Compose, do the following: 6. Test the installation. $ docker-compose --version - docker-compose version: 1.4.0 + docker-compose version: 1.4.1 ## Upgrading From bdfb21f0171ffa175be5414b192e9b88f7775d04 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 20:46:22 -0400 Subject: [PATCH 179/337] Fixes #189 - stacktrace when ctrl-c stops logs Signed-off-by: Daniel Nephin --- compose/cli/multiplexer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index b502c351b7..4c73c6cdc6 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from threading import Thread +from six.moves import _thread as thread + try: from Queue import Queue, Empty except ImportError: @@ -38,6 +40,9 @@ class Multiplexer(object): yield item except Empty: pass + # See https://github.com/docker/compose/issues/189 + except thread.error: + raise KeyboardInterrupt() def _init_readers(self): for iterator in self.iterators: From bbc8765343f7824e2107bb78acb8814de3f1cb4e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 16 Sep 2015 12:38:59 +0100 Subject: [PATCH 180/337] Fix typo in docs/index.md Signed-off-by: Aanand Prasad --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 3180d7df0a..0c919e488b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -139,7 +139,7 @@ This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the current directory on the host to ``/code` inside the container allowing you to modify the code without having to rebuild the image. +* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. * Links the web container to the Redis service. The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. From fb83b4c6a40406c968c6c6723259e25d10cc2237 Mon Sep 17 00:00:00 2001 From: Zachary Jaffee Date: Wed, 16 Sep 2015 11:01:43 -0400 Subject: [PATCH 181/337] updated wordpress format syntax Signed-off-by: Zachary Jaffee --- docs/completion.md | 2 +- docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 2 +- docs/index.md | 2 +- docs/install.md | 2 +- docs/production.md | 2 +- docs/rails.md | 2 +- docs/wordpress.md | 20 ++++++++++---------- docs/yml.md | 2 +- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 7b8a6733e5..bf8d15551e 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -63,7 +63,7 @@ Enjoy working with Compose faster and with less typos! - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/django.md b/docs/django.md index 7e476b3569..e52f50301d 100644 --- a/docs/django.md +++ b/docs/django.md @@ -128,7 +128,7 @@ example, run `docker-compose up` and in another terminal run: - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/env.md b/docs/env.md index 8ead34f01f..a8e6e214ce 100644 --- a/docs/env.md +++ b/docs/env.md @@ -43,7 +43,7 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose command line completion](completion.md) diff --git a/docs/extends.md b/docs/extends.md index 18a072a82d..7b4d5b2093 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -357,7 +357,7 @@ locally-defined bindings taking precedence: - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose command line completion](completion.md) diff --git a/docs/index.md b/docs/index.md index 3180d7df0a..0112d0aa47 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,7 +52,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/install.md b/docs/install.md index 371d0a903f..b293246770 100644 --- a/docs/install.md +++ b/docs/install.md @@ -96,7 +96,7 @@ To uninstall Docker Compose if you installed using `pip`: - [User guide](/) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/production.md b/docs/production.md index 5a3a07e8e2..29e3fd34ec 100644 --- a/docs/production.md +++ b/docs/production.md @@ -88,7 +88,7 @@ guide. - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/rails.md b/docs/rails.md index 186f9b2bf2..0a164ca75e 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -126,7 +126,7 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index ab22e2a0df..8de5a26441 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -1,7 +1,7 @@ -# Quickstart Guide: Compose and Wordpress +# Quickstart Guide: Compose and WordPress -You can use Compose to easily run Wordpress in an isolated environment built +You can use Compose to easily run WordPress in an isolated environment built with Docker containers. ## Define the project -First, [Install Compose](install.md) and then download Wordpress into the +First, [Install Compose](install.md) and then download WordPress into the current directory: $ curl https://wordpress.org/latest.tar.gz | tar -xvzf - @@ -36,7 +36,7 @@ your Dockerfile should be: ADD . /code This tells Docker how to build an image defining a container that contains PHP -and Wordpress. +and WordPress. Next you'll create a `docker-compose.yml` file that will start your web service and a separate MySQL instance: @@ -56,7 +56,7 @@ and a separate MySQL instance: MYSQL_DATABASE: wordpress Two supporting files are needed to get this working - first, `wp-config.php` is -the standard Wordpress config file with a single change to point the database +the standard WordPress config file with a single change to point the database configuration at the `db` container: Date: Tue, 18 Aug 2015 16:43:19 +0100 Subject: [PATCH 182/337] Use docker.client.create_host_config create_host_config from docker.utils will be deprecated so that the new create_host_config has access to the _version so we can ensure that network_mode only gets set to 'default' by default if the version is high enough and won't explode. Signed-off-by: Mazz Mosley --- compose/service.py | 3 +-- tests/integration/service_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 7e035e29f0..6d3df1f7e3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -11,7 +11,6 @@ from operator import attrgetter import enum import six from docker.errors import APIError -from docker.utils import create_host_config from docker.utils import LogConfig from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port @@ -678,7 +677,7 @@ class Service(object): devices = options.get('devices', None) - return create_host_config( + return self.client.create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=port_bindings, binds=options.get('binds'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 17fd0aaf1b..040098c9e7 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -813,6 +813,13 @@ class ServiceTest(DockerClientTestCase): for k, v in {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}.items(): self.assertEqual(env[k], v) + def test_with_high_enough_api_version_we_get_default_network_mode(self): + # TODO: remove this test once minimum docker version is 1.8.x + with mock.patch.object(self.client, '_version', '1.20'): + service = self.create_service('web') + service_config = service._get_container_host_config({}) + self.assertEquals(service_config['NetworkMode'], 'default') + def test_labels(self): labels_dict = { 'com.example.description': "Accounting webapp", From 6f6c04b5c938a7ee510d63bb36912a2e7513cb71 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 16 Sep 2015 12:02:58 +0100 Subject: [PATCH 183/337] Test what we are sending, not what we get This is a unit test and we are mocking the client. The method to get the create_config_host now lives on the client, so we mock that too. So we can test to the boundary that the method is called with the arguments we expect. Signed-off-by: Mazz Mosley --- tests/unit/service_test.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index de973339b2..7ba630fb44 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,7 +2,6 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker -from docker.utils import LogConfig from .. import mock from .. import unittest @@ -108,19 +107,33 @@ class ServiceTest(unittest.TestCase): self.assertFalse('domainname' in opts, 'domainname') def test_memory_swap_limit(self): + self.mock_client.create_host_config.return_value = {} + service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) - opts = service._get_container_create_options({'some': 'overrides'}, 1) - self.assertEqual(opts['host_config']['MemorySwap'], 2000000000) - self.assertEqual(opts['host_config']['Memory'], 1000000000) + service._get_container_create_options({'some': 'overrides'}, 1) + + self.assertTrue(self.mock_client.create_host_config.called) + self.assertEqual( + self.mock_client.create_host_config.call_args[1]['mem_limit'], + 1000000000 + ) + self.assertEqual( + self.mock_client.create_host_config.call_args[1]['memswap_limit'], + 2000000000 + ) def test_log_opt(self): + self.mock_client.create_host_config.return_value = {} + log_opt = {'syslog-address': 'tcp://192.168.0.42:123'} service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) - opts = service._get_container_create_options({'some': 'overrides'}, 1) + service._get_container_create_options({'some': 'overrides'}, 1) - self.assertIsInstance(opts['host_config']['LogConfig'], LogConfig) - self.assertEqual(opts['host_config']['LogConfig'].type, 'syslog') - self.assertEqual(opts['host_config']['LogConfig'].config, log_opt) + self.assertTrue(self.mock_client.create_host_config.called) + self.assertEqual( + self.mock_client.create_host_config.call_args[1]['log_config'], + {'Type': 'syslog', 'Config': {'syslog-address': 'tcp://192.168.0.42:123'}} + ) def test_split_domainname_fqdn(self): service = Service( From 39ba2c5a7cb5a4f7cec1e5a28bd43dc95492b22d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 17 Sep 2015 15:33:58 +0100 Subject: [PATCH 184/337] Fix leaky tests It was mocking self.client but relying on the call to utils.create_host_config which was not mocked. So now that function has moved to also be on self.client we need to redefine the test boundary, up to where we would call docker-py, not the result of docker-py. Signed-off-by: Mazz Mosley --- tests/unit/cli_test.py | 13 +++++++++---- tests/unit/service_test.py | 10 +++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 1fd9f529ed..d12f41955d 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -144,8 +144,11 @@ class CLITestCase(unittest.TestCase): '--rm': None, '--name': None, }) - _, _, call_kwargs = mock_client.create_container.mock_calls[0] - self.assertEquals(call_kwargs['host_config']['RestartPolicy']['Name'], 'always') + + self.assertEquals( + mock_client.create_host_config.call_args[1]['restart_policy']['Name'], + 'always' + ) command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) @@ -170,8 +173,10 @@ class CLITestCase(unittest.TestCase): '--rm': True, '--name': None, }) - _, _, call_kwargs = mock_client.create_container.mock_calls[0] - self.assertFalse('RestartPolicy' in call_kwargs['host_config']) + + self.assertFalse( + mock_client.create_host_config.call_args[1].get('restart_policy') + ) def test_command_manula_and_service_ports_together(self): command = TopLevelCommand() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7ba630fb44..5f7ae94875 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -543,13 +543,13 @@ class ServiceVolumesTest(unittest.TestCase): } } - create_options = service._get_container_create_options( + service._get_container_create_options( override_options={}, number=1, ) self.assertEqual( - set(create_options['host_config']['Binds']), + set(self.mock_client.create_host_config.call_args[1]['binds']), set([ '/host/path:/data1:rw', '/host/path:/data2:rw', @@ -581,14 +581,14 @@ class ServiceVolumesTest(unittest.TestCase): }, } - create_options = service._get_container_create_options( + service._get_container_create_options( override_options={}, number=1, previous_container=Container(self.mock_client, {'Id': '123123123'}), ) self.assertEqual( - create_options['host_config']['Binds'], + self.mock_client.create_host_config.call_args[1]['binds'], ['/mnt/sda1/host/path:/data:rw'], ) @@ -613,4 +613,4 @@ class ServiceVolumesTest(unittest.TestCase): ).create_container() self.assertEqual(len(create_calls), 1) - self.assertEqual(create_calls[0][1]['host_config']['Binds'], volumes) + self.assertEqual(self.mock_client.create_host_config.call_args[1]['binds'], volumes) From 9be748f85c208da8c90636db400e418a5b0f353b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 14 Sep 2015 18:42:53 -0400 Subject: [PATCH 185/337] Clean before doing a build so that we don't include stale build artifacts in the binaries. Signed-off-by: Daniel Nephin --- script/build-linux | 2 ++ script/build-osx | 2 ++ script/clean | 3 +++ 3 files changed, 7 insertions(+) diff --git a/script/build-linux b/script/build-linux index 4fdf1d926f..7d89bd1e5b 100755 --- a/script/build-linux +++ b/script/build-linux @@ -2,6 +2,8 @@ set -ex +./script/clean + TAG="docker-compose" docker build -t "$TAG" . docker run \ diff --git a/script/build-osx b/script/build-osx index e1cc7038ac..11b6ecc694 100755 --- a/script/build-osx +++ b/script/build-osx @@ -3,7 +3,9 @@ set -ex PATH="/usr/local/bin:$PATH" +./script/clean rm -rf venv + virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt diff --git a/script/clean b/script/clean index 07a9cff14d..08ba551ae9 100755 --- a/script/clean +++ b/script/clean @@ -1,3 +1,6 @@ #!/bin/sh +set -e + find . -type f -name '*.pyc' -delete +find -name __pycache__ -delete rm -rf docs/_site build dist docker-compose.egg-info From 2121f5117ea035c83d4ace97fad8f2db6582afc9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 18:20:35 -0400 Subject: [PATCH 186/337] Add docopt support for multiple files Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 61461ae7be..3dd0c9fae3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -96,7 +96,7 @@ class TopLevelCommand(Command): """Define and run multi-container applications with Docker. Usage: - docker-compose [options] [COMMAND] [ARGS...] + docker-compose [-f=...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: From 258d0fa0c660813d8b6b3d8d17731cc56a5da321 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 19:51:50 -0400 Subject: [PATCH 187/337] Remove some functions from Command class Signed-off-by: Daniel Nephin --- compose/cli/command.py | 82 +++++++++++++++++++++++------------------- tests/unit/cli_test.py | 39 +++++++++----------- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 67176df271..70b129d296 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -55,53 +55,61 @@ class Command(DocoptCommand): log.warn('The FIG_FILE environment variable is deprecated.') log.warn('Please use COMPOSE_FILE instead.') - explicit_config_path = options.get('--file') or os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE') - project = self.get_project( + explicit_config_path = ( + options.get('--file') or + os.environ.get('COMPOSE_FILE') or + os.environ.get('FIG_FILE')) + + project = get_project( + self.base_dir, explicit_config_path, project_name=options.get('--project-name'), verbose=options.get('--verbose')) handler(project, command_options) - def get_client(self, verbose=False): - client = docker_client() - if verbose: - version_info = six.iteritems(client.version()) - log.info("Compose version %s", __version__) - log.info("Docker base_url: %s", client.base_url) - log.info("Docker version: %s", - ", ".join("%s=%s" % item for item in version_info)) - return verbose_proxy.VerboseProxy('docker', client) - return client - def get_project(self, config_path=None, project_name=None, verbose=False): - config_details = config.find(self.base_dir, config_path) +def get_client(verbose=False): + client = docker_client() + if verbose: + version_info = six.iteritems(client.version()) + log.info("Compose version %s", __version__) + log.info("Docker base_url: %s", client.base_url) + log.info("Docker version: %s", + ", ".join("%s=%s" % item for item in version_info)) + return verbose_proxy.VerboseProxy('docker', client) + return client - try: - return Project.from_dicts( - self.get_project_name(config_details.working_dir, project_name), - config.load(config_details), - self.get_client(verbose=verbose)) - except ConfigError as e: - raise errors.UserError(six.text_type(e)) - def get_project_name(self, working_dir, project_name=None): - def normalize_name(name): - return re.sub(r'[^a-z0-9]', '', name.lower()) +def get_project(base_dir, config_path=None, project_name=None, verbose=False): + config_details = config.find(base_dir, config_path) - if 'FIG_PROJECT_NAME' in os.environ: - log.warn('The FIG_PROJECT_NAME environment variable is deprecated.') - log.warn('Please use COMPOSE_PROJECT_NAME instead.') + try: + return Project.from_dicts( + get_project_name(config_details.working_dir, project_name), + config.load(config_details), + get_client(verbose=verbose)) + except ConfigError as e: + raise errors.UserError(six.text_type(e)) - project_name = ( - project_name or - os.environ.get('COMPOSE_PROJECT_NAME') or - os.environ.get('FIG_PROJECT_NAME')) - if project_name is not None: - return normalize_name(project_name) - project = os.path.basename(os.path.abspath(working_dir)) - if project: - return normalize_name(project) +def get_project_name(working_dir, project_name=None): + def normalize_name(name): + return re.sub(r'[^a-z0-9]', '', name.lower()) - return 'default' + if 'FIG_PROJECT_NAME' in os.environ: + log.warn('The FIG_PROJECT_NAME environment variable is deprecated.') + log.warn('Please use COMPOSE_PROJECT_NAME instead.') + + project_name = ( + project_name or + os.environ.get('COMPOSE_PROJECT_NAME') or + os.environ.get('FIG_PROJECT_NAME')) + if project_name is not None: + return normalize_name(project_name) + + project = os.path.basename(os.path.abspath(working_dir)) + if project: + return normalize_name(project) + + return 'default' diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index d12f41955d..321df97a53 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -4,9 +4,12 @@ from __future__ import unicode_literals import os import docker +import py from .. import mock from .. import unittest +from compose.cli.command import get_project +from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand @@ -14,55 +17,45 @@ from compose.service import Service class CLITestCase(unittest.TestCase): - def test_default_project_name(self): - cwd = os.getcwd() - try: - os.chdir('tests/fixtures/simple-composefile') - command = TopLevelCommand() - project_name = command.get_project_name('.') + def test_default_project_name(self): + test_dir = py._path.local.LocalPath('tests/fixtures/simple-composefile') + with test_dir.as_cwd(): + project_name = get_project_name('.') self.assertEquals('simplecomposefile', project_name) - finally: - os.chdir(cwd) def test_project_name_with_explicit_base_dir(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/simple-composefile' - project_name = command.get_project_name(command.base_dir) + base_dir = 'tests/fixtures/simple-composefile' + project_name = get_project_name(base_dir) self.assertEquals('simplecomposefile', project_name) def test_project_name_with_explicit_uppercase_base_dir(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/UpperCaseDir' - project_name = command.get_project_name(command.base_dir) + base_dir = 'tests/fixtures/UpperCaseDir' + project_name = get_project_name(base_dir) self.assertEquals('uppercasedir', project_name) def test_project_name_with_explicit_project_name(self): - command = TopLevelCommand() name = 'explicit-project-name' - project_name = command.get_project_name(None, project_name=name) + project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) def test_project_name_from_environment_old_var(self): - command = TopLevelCommand() name = 'namefromenv' with mock.patch.dict(os.environ): os.environ['FIG_PROJECT_NAME'] = name - project_name = command.get_project_name(None) + project_name = get_project_name(None) self.assertEquals(project_name, name) def test_project_name_from_environment_new_var(self): - command = TopLevelCommand() name = 'namefromenv' with mock.patch.dict(os.environ): os.environ['COMPOSE_PROJECT_NAME'] = name - project_name = command.get_project_name(None) + project_name = get_project_name(None) self.assertEquals(project_name, name) def test_get_project(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/longer-filename-composefile' - project = command.get_project() + base_dir = 'tests/fixtures/longer-filename-composefile' + project = get_project(base_dir) self.assertEqual(project.name, 'longerfilenamecomposefile') self.assertTrue(project.client) self.assertTrue(project.services) From 10b3188214fc6716387339bb0146d8d901962e93 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 20:18:45 -0400 Subject: [PATCH 188/337] Support multiple config files Signed-off-by: Daniel Nephin --- compose/cli/command.py | 22 +++++---- compose/config/config.py | 66 +++++++++++++++++---------- tests/unit/config_test.py | 96 +++++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 79 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 70b129d296..2120ec4db5 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -51,24 +51,26 @@ class Command(DocoptCommand): handler(None, command_options) return - if 'FIG_FILE' in os.environ: - log.warn('The FIG_FILE environment variable is deprecated.') - log.warn('Please use COMPOSE_FILE instead.') - - explicit_config_path = ( - options.get('--file') or - os.environ.get('COMPOSE_FILE') or - os.environ.get('FIG_FILE')) - project = get_project( self.base_dir, - explicit_config_path, + get_config_path(options.get('--file')), project_name=options.get('--project-name'), verbose=options.get('--verbose')) handler(project, command_options) +def get_config_path(file_option): + if file_option: + return file_option + + if 'FIG_FILE' in os.environ: + log.warn('The FIG_FILE environment variable is deprecated.') + log.warn('Please use COMPOSE_FILE instead.') + + return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')] + + def get_client(verbose=False): client = docker_client() if verbose: diff --git a/compose/config/config.py b/compose/config/config.py index 840a28a1b5..204f70b666 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ import logging import os import sys from collections import namedtuple +from functools import reduce import six import yaml @@ -88,18 +89,24 @@ PATH_START_CHARS = [ log = logging.getLogger(__name__) -ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename') +ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs') + +ConfigFile = namedtuple('ConfigFile', 'filename config') -def find(base_dir, filename): - if filename == '-': - return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None) +def find(base_dir, filenames): + if filenames == ['-']: + return ConfigDetails( + os.getcwd(), + [ConfigFile(None, yaml.safe_load(sys.stdin))]) - if filename: - filename = os.path.join(base_dir, filename) + if filenames: + filenames = [os.path.join(base_dir, f) for f in filenames] else: - filename = get_config_path(base_dir) - return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename) + filenames = [get_config_path(base_dir)] + return ConfigDetails( + os.path.dirname(filenames[0]), + [ConfigFile(f, load_yaml(f)) for f in filenames]) def get_config_path(base_dir): @@ -133,29 +140,40 @@ def pre_process_config(config): Pre validation checks and processing of the config file to interpolate env vars returning a config dict ready to be tested against the schema. """ - config = interpolate_environment_variables(config) - return config + return interpolate_environment_variables(config) def load(config_details): - config, working_dir, filename = config_details + working_dir, configs = config_details - processed_config = pre_process_config(config) - validate_against_fields_schema(processed_config) - - service_dicts = [] - - for service_name, service_dict in list(processed_config.items()): - loader = ServiceLoader( - working_dir=working_dir, - filename=filename, - service_name=service_name, - service_dict=service_dict) + def build_service(filename, service_name, service_dict): + loader = ServiceLoader(working_dir, filename, service_name, service_dict) service_dict = loader.make_service_dict() validate_paths(service_dict) - service_dicts.append(service_dict) + return service_dict - return service_dicts + def load_file(filename, config): + processed_config = pre_process_config(config) + validate_against_fields_schema(processed_config) + return [ + build_service(filename, name, service_config) + for name, service_config in processed_config.items() + ] + + def merge_services(base, override): + return { + name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + for name in set(base) | set(override) + } + + def combine_configs(override, base): + service_dicts = load_file(base.filename, base.config) + if not override: + return service_dicts + + return merge_service_dicts(base.config, override.config) + + return reduce(combine_configs, configs, None) class ServiceLoader(object): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index ff80270e6d..0347e443f8 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -26,10 +26,16 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def build_config_details(contents, working_dir, filename): + return config.ConfigDetails( + working_dir, + [config.ConfigFile(filename, contents)]) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox'}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, @@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase): def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( {'web': 'busybox:latest'}, 'working_dir', 'filename.yml' @@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: config.load( - config.ConfigDetails( + build_config_details( {invalid_name: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service name: 1 needs to be a string, eg '1'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {1: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase): def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( - config.ConfigDetails( + build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', 'common.yml' @@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': invalid_ports}}, 'working_dir', 'filename.yml' @@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase): valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] for ports in valid_ports: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': ports}}, 'working_dir', 'filename.yml' @@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "(did you mean 'privileged'?)" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'privilige': 'something'}, }, @@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' has both an image and build path specified." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'build': '.'}, }, @@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'links': 'an_link'}, }, @@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Top level object needs to be a dictionary." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( ['foo', 'lol'], 'tests/fixtures/extends', 'filename.yml' @@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "has non-unique elements" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']} }, @@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg += ", which is an invalid type, it should be a string" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'command': [1]} }, @@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, 'working_dir', 'filename.yml' @@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': 'somehost:162.242.195.82' @@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': [ @@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase): expose_values = [["8000"], [8000]] for expose in expose_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'expose': expose @@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase): entrypoint_values = [["sh"], "sh"] for entrypoint in entrypoint_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'entrypoint': entrypoint @@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase): def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) os.environ.pop('BAR', None) - config_details = config.ConfigDetails( - config={ + config_details = build_config_details( + { 'web': { 'image': '${FOO}', 'command': '${BAR}', 'container_name': '${BAR}', }, }, - working_dir='.', - filename=None, + '.', + None, ) with mock.patch('compose.config.interpolation.log') as log: @@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase): def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': '${'}}, 'working_dir', 'filename.yml' @@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase): def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' d = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - working_dir='.', - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, ) )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) @@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase): ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'memswap_limit': 2000000}, }, @@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_validation_with_correct_memswap_values(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}}, 'tests/fixtures/extends', 'common.yml' @@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_memswap_can_be_a_string(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}}, 'tests/fixtures/extends', 'common.yml' @@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase): os.environ['CONTAINERENV'] = '/host/tmp' service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) def load_from_filename(filename): - return config.load(config.find('.', filename)) + return config.load(config.find('.', [filename])) class ExtendsTest(unittest.TestCase): @@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_empty_dictionary(self): with self.assertRaisesRegexp(ConfigurationError, 'service'): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {}}, }, @@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_missing_service_key(self): with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}}, }, @@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_valid_config(self): service = config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}}, }, @@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase): def test_nonexistent_path(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'build': 'nonexistent.path'}, }, From c0c9a7c1e4d22980afb6e22817a960f7424f0eae Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 20:50:31 -0400 Subject: [PATCH 189/337] Update integration tests for multiple file support Signed-off-by: Daniel Nephin --- compose/cli/command.py | 3 ++- tests/integration/cli_test.py | 7 ++++--- tests/integration/project_test.py | 7 +++++-- tests/integration/state_test.py | 8 +++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 2120ec4db5..950cb166e6 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -68,7 +68,8 @@ def get_config_path(file_option): log.warn('The FIG_FILE environment variable is deprecated.') log.warn('Please use COMPOSE_FILE instead.') - return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')] + config_file = os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE') + return [config_file] if config_file else None def get_client(verbose=False): diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 4a80d33695..8688fb8b4d 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -9,6 +9,7 @@ from six import StringIO from .. import mock from .testcases import DockerClientTestCase +from compose.cli.command import get_project from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.project import NoSuchService @@ -38,7 +39,7 @@ class CLITestCase(DockerClientTestCase): if hasattr(self, '_project'): return self._project - return self.command.get_project() + return get_project(self.command.base_dir) def test_help(self): old_base_dir = self.command.base_dir @@ -72,7 +73,7 @@ class CLITestCase(DockerClientTestCase): def test_ps_alternate_composefile(self, mock_stdout): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') - self._project = self.command.get_project(config_path) + self._project = get_project(self.command.base_dir, [config_path]) self.command.base_dir = 'tests/fixtures/multiple-composefiles' self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) @@ -571,7 +572,7 @@ class CLITestCase(DockerClientTestCase): def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') self.command.dispatch(['-f', config_path, 'up', '-d'], None) - self._project = self.command.get_project(config_path) + self._project = get_project(self.command.base_dir, [config_path]) containers = self.project.containers(stopped=True) self.assertEqual(len(containers), 1) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ad49ad10a8..bd7ecccbe8 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase -from compose import config +from compose.config import config from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project @@ -9,7 +9,10 @@ from compose.service import ConvergenceStrategy def build_service_dicts(service_config): - return config.load(config.ConfigDetails(service_config, 'working_dir', None)) + return config.load( + config.ConfigDetails( + 'working_dir', + [config.ConfigFile(None, service_config)])) class ProjectTest(DockerClientTestCase): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 93d0572a08..ef7276bd8d 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -9,7 +9,7 @@ import shutil import tempfile from .testcases import DockerClientTestCase -from compose import config +from compose.config import config from compose.const import LABEL_CONFIG_HASH from compose.project import Project from compose.service import ConvergenceStrategy @@ -24,11 +24,13 @@ class ProjectTestCase(DockerClientTestCase): return set(project.containers(stopped=True)) def make_project(self, cfg): + details = config.ConfigDetails( + 'working_dir', + [config.ConfigFile(None, cfg)]) return Project.from_dicts( name='composetest', client=self.client, - service_dicts=config.load(config.ConfigDetails(cfg, 'working_dir', None)) - ) + service_dicts=config.load(details)) class BasicProjectTest(ProjectTestCase): From 831276f53163c0999ec635d92629e6e1b4ba2683 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 14 Sep 2015 21:30:32 -0400 Subject: [PATCH 190/337] Move config_test to the correct package name. Signed-off-by: Daniel Nephin --- tests/unit/config/__init__.py | 0 tests/unit/{ => config}/config_test.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/unit/config/__init__.py rename tests/unit/{ => config}/config_test.py (99%) diff --git a/tests/unit/config/__init__.py b/tests/unit/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/config_test.py b/tests/unit/config/config_test.py similarity index 99% rename from tests/unit/config_test.py rename to tests/unit/config/config_test.py index 0347e443f8..3542f272b8 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config/config_test.py @@ -280,7 +280,7 @@ class ConfigTest(unittest.TestCase): def test_logs_warning_for_boolean_in_environment(self, mock_logging): expected_warning_msg = "Warning: There is a boolean value, True in the 'environment' key." config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'environment': {'SHOW_STUFF': True} @@ -298,7 +298,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'environment': {'---': 'nope'} From 89be7f1fa76f53dbc082715eadec65e08f992e8a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 14 Sep 2015 21:35:41 -0400 Subject: [PATCH 191/337] Unit tests for multiple files Signed-off-by: Daniel Nephin --- compose/config/config.py | 8 ++++--- tests/unit/config/config_test.py | 41 ++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 204f70b666..058183d97d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -166,14 +166,16 @@ def load(config_details): for name in set(base) | set(override) } - def combine_configs(override, base): + def combine_configs(base, override): service_dicts = load_file(base.filename, base.config) if not override: return service_dicts - return merge_service_dicts(base.config, override.config) + return ConfigFile( + override.filename, + merge_services(base.config, override.config)) - return reduce(combine_configs, configs, None) + return reduce(combine_configs, configs + [None]) class ServiceLoader(object): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3542f272b8..60f4bbe222 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5,10 +5,10 @@ import shutil import tempfile from operator import itemgetter -from .. import mock -from .. import unittest from compose.config import config from compose.config.errors import ConfigurationError +from tests import mock +from tests import unittest def make_service_dict(name, service_dict, working_dir, filename=None): @@ -92,6 +92,43 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_with_multiple_files(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': { + 'image': 'example/web', + 'links': ['db'], + }, + 'db': { + 'image': 'example/db', + }, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'web': { + 'build': '/', + 'volumes': ['/home/user/project:/code'], + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + service_dicts = config.load(details) + expected = [ + { + 'name': 'web', + 'build': '/', + 'links': ['db'], + 'volumes': ['/home/user/project:/code'], + }, + { + 'name': 'db', + 'image': 'example/db', + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From fe5daf860dfb341ba894d79d225689ed2e981064 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:17:27 -0400 Subject: [PATCH 192/337] Move find_candidates_in_parent_dirs() into a config module so that config doesn't import from cli. Signed-off-by: Daniel Nephin --- compose/cli/utils.py | 19 ------------------- compose/config/config.py | 24 +++++++++++++++++++++--- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 0b7ac683d1..0a4416c0f3 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -36,25 +36,6 @@ def yesno(prompt, default=None): return None -def find_candidates_in_parent_dirs(filenames, path): - """ - Given a directory path to start, looks for filenames in the - directory, and then each parent directory successively, - until found. - - Returns tuple (candidates, path). - """ - candidates = [filename for filename in filenames - if os.path.exists(os.path.join(path, filename))] - - if len(candidates) == 0: - parent_dir = os.path.join(path, '..') - if os.path.abspath(parent_dir) != os.path.abspath(path): - return find_candidates_in_parent_dirs(filenames, parent_dir) - - return (candidates, path) - - def split_buffer(reader, separator): """ Given a generator which yields strings and a separator string, diff --git a/compose/config/config.py b/compose/config/config.py index 058183d97d..2e4d0a7511 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -17,7 +17,6 @@ from .validation import validate_extended_service_exists from .validation import validate_extends_file_path from .validation import validate_service_names from .validation import validate_top_level_object -from compose.cli.utils import find_candidates_in_parent_dirs DOCKER_CONFIG_KEYS = [ @@ -103,13 +102,13 @@ def find(base_dir, filenames): if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] else: - filenames = [get_config_path(base_dir)] + filenames = get_default_config_path(base_dir) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) -def get_config_path(base_dir): +def get_default_config_path(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) if len(candidates) == 0: @@ -133,6 +132,25 @@ def get_config_path(base_dir): return os.path.join(path, winner) +def find_candidates_in_parent_dirs(filenames, path): + """ + Given a directory path to start, looks for filenames in the + directory, and then each parent directory successively, + until found. + + Returns tuple (candidates, path). + """ + candidates = [filename for filename in filenames + if os.path.exists(os.path.join(path, filename))] + + if len(candidates) == 0: + parent_dir = os.path.join(path, '..') + if os.path.abspath(parent_dir) != os.path.abspath(path): + return find_candidates_in_parent_dirs(filenames, parent_dir) + + return (candidates, path) + + @validate_top_level_object @validate_service_names def pre_process_config(config): From 39ae85db8ad4aa6429d6c4863d67b21d1c93aac7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:43:18 -0400 Subject: [PATCH 193/337] Support a default docker-compose.override.yml for overrides Signed-off-by: Daniel Nephin --- compose/config/config.py | 16 +++++--- .../docker-compose.override.yml | 6 +++ .../override-files/docker-compose.yml | 10 +++++ tests/fixtures/override-files/extra.yml | 9 +++++ tests/integration/cli_test.py | 39 ++++++++++++++++++- tests/unit/config/config_test.py | 3 +- 6 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/override-files/docker-compose.override.yml create mode 100644 tests/fixtures/override-files/docker-compose.yml create mode 100644 tests/fixtures/override-files/extra.yml diff --git a/compose/config/config.py b/compose/config/config.py index 2e4d0a7511..3ecdd29d7b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -77,6 +77,7 @@ SUPPORTED_FILENAMES = [ 'fig.yaml', ] +DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' PATH_START_CHARS = [ '/', @@ -102,16 +103,16 @@ def find(base_dir, filenames): if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] else: - filenames = get_default_config_path(base_dir) + filenames = get_default_config_files(base_dir) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) -def get_default_config_path(base_dir): +def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) - if len(candidates) == 0: + if not candidates: raise ComposeFileNotFound(SUPPORTED_FILENAMES) winner = candidates[0] @@ -129,7 +130,12 @@ def get_default_config_path(base_dir): log.warn("%s is deprecated and will not be supported in future. " "Please rename your config file to docker-compose.yml\n" % winner) - return os.path.join(path, winner) + return [os.path.join(path, winner)] + get_default_override_file(path) + + +def get_default_override_file(path): + override_filename = os.path.join(path, DEFAULT_OVERRIDE_FILENAME) + return [override_filename] if os.path.exists(override_filename) else [] def find_candidates_in_parent_dirs(filenames, path): @@ -143,7 +149,7 @@ def find_candidates_in_parent_dirs(filenames, path): candidates = [filename for filename in filenames if os.path.exists(os.path.join(path, filename))] - if len(candidates) == 0: + if not candidates: parent_dir = os.path.join(path, '..') if os.path.abspath(parent_dir) != os.path.abspath(path): return find_candidates_in_parent_dirs(filenames, parent_dir) diff --git a/tests/fixtures/override-files/docker-compose.override.yml b/tests/fixtures/override-files/docker-compose.override.yml new file mode 100644 index 0000000000..a03d3d6f5f --- /dev/null +++ b/tests/fixtures/override-files/docker-compose.override.yml @@ -0,0 +1,6 @@ + +web: + command: "top" + +db: + command: "top" diff --git a/tests/fixtures/override-files/docker-compose.yml b/tests/fixtures/override-files/docker-compose.yml new file mode 100644 index 0000000000..8eb43ddb06 --- /dev/null +++ b/tests/fixtures/override-files/docker-compose.yml @@ -0,0 +1,10 @@ + +web: + image: busybox:latest + command: "sleep 200" + links: + - db + +db: + image: busybox:latest + command: "sleep 200" diff --git a/tests/fixtures/override-files/extra.yml b/tests/fixtures/override-files/extra.yml new file mode 100644 index 0000000000..7b3ade9c2d --- /dev/null +++ b/tests/fixtures/override-files/extra.yml @@ -0,0 +1,9 @@ + +web: + links: + - db + - other + +other: + image: busybox:latest + command: "top" diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 8688fb8b4d..33fdda3bec 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -549,7 +549,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3002), "0.0.0.0:49153") def test_port_with_scale(self): - self.command.base_dir = 'tests/fixtures/ports-composefile-scale' self.command.dispatch(['scale', 'simple=2'], None) containers = sorted( @@ -593,6 +592,44 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + def test_up_with_default_override_file(self): + self.command.base_dir = 'tests/fixtures/override-files' + self.command.dispatch(['up', '-d'], None) + + containers = self.project.containers() + self.assertEqual(len(containers), 2) + + web, db = containers + self.assertEqual(web.human_readable_command, 'top') + self.assertEqual(db.human_readable_command, 'top') + + def test_up_with_multiple_files(self): + self.command.base_dir = 'tests/fixtures/override-files' + config_paths = [ + 'docker-compose.yml', + 'docker-compose.override.yml', + 'extra.yml', + + ] + self._project = get_project(self.command.base_dir, config_paths) + self.command.dispatch( + [ + '-f', config_paths[0], + '-f', config_paths[1], + '-f', config_paths[2], + 'up', '-d', + ], + None) + + containers = self.project.containers() + self.assertEqual(len(containers), 3) + + web, other, db = containers + self.assertEqual(web.human_readable_command, 'top') + self.assertTrue({'db', 'other'} <= set(web.links())) + self.assertEqual(db.human_readable_command, 'top') + self.assertEqual(other.human_readable_command, 'top') + def test_up_with_extends(self): self.command.base_dir = 'tests/fixtures/extends' self.command.dispatch(['up', '-d'], None) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 60f4bbe222..38eb3de23c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1213,6 +1213,7 @@ def get_config_filename_for_files(filenames, subdir=None): base_dir = tempfile.mkdtemp(dir=project_dir) else: base_dir = project_dir - return os.path.basename(config.get_config_path(base_dir)) + filename, = config.get_default_config_files(base_dir) + return os.path.basename(filename) finally: shutil.rmtree(project_dir) From be0611bf3e435c9431d023b7f1fdef0e487554b0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:47:33 -0400 Subject: [PATCH 194/337] Cleanup get_default_config_files tests. Signed-off-by: Daniel Nephin --- tests/unit/config/config_test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 38eb3de23c..79864ec784 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1167,7 +1167,7 @@ class BuildPathTest(unittest.TestCase): self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) -class GetConfigPathTestCase(unittest.TestCase): +class GetDefaultConfigFilesTestCase(unittest.TestCase): files = [ 'docker-compose.yml', @@ -1177,25 +1177,21 @@ class GetConfigPathTestCase(unittest.TestCase): ] def test_get_config_path_default_file_in_basedir(self): - files = self.files - self.assertEqual('docker-compose.yml', get_config_filename_for_files(files[0:])) - self.assertEqual('docker-compose.yaml', get_config_filename_for_files(files[1:])) - self.assertEqual('fig.yml', get_config_filename_for_files(files[2:])) - self.assertEqual('fig.yaml', get_config_filename_for_files(files[3:])) + for index, filename in enumerate(self.files): + self.assertEqual( + filename, + get_config_filename_for_files(self.files[index:])) with self.assertRaises(config.ComposeFileNotFound): get_config_filename_for_files([]) def test_get_config_path_default_file_in_parent_dir(self): """Test with files placed in the subdir""" - files = self.files def get_config_in_subdir(files): return get_config_filename_for_files(files, subdir=True) - self.assertEqual('docker-compose.yml', get_config_in_subdir(files[0:])) - self.assertEqual('docker-compose.yaml', get_config_in_subdir(files[1:])) - self.assertEqual('fig.yml', get_config_in_subdir(files[2:])) - self.assertEqual('fig.yaml', get_config_in_subdir(files[3:])) + for index, filename in enumerate(self.files): + self.assertEqual(filename, get_config_in_subdir(self.files[index:])) with self.assertRaises(config.ComposeFileNotFound): get_config_in_subdir([]) From fd75e4bf6385d33165f1c91af2f63d9a8201e530 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 16 Sep 2015 15:03:55 -0400 Subject: [PATCH 195/337] Update docs about using multiple -f arguments Signed-off-by: Daniel Nephin --- docs/reference/docker-compose.md | 62 ++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index b43055fbea..32fcbe7064 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -14,7 +14,7 @@ weight=-2 ``` Usage: - docker-compose [options] [COMMAND] [ARGS...] + docker-compose [-f=...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: @@ -41,20 +41,62 @@ Commands: unpause Unpause services up Create and start containers migrate-to-labels Recreate containers to add labels + version Show the Docker-Compose version information ``` -The Docker Compose binary. You use this command to build and manage multiple services in Docker containers. +The Docker Compose binary. You use this command to build and manage multiple +services in Docker containers. -Use the `-f` flag to specify the location of a Compose configuration file. This -flag is optional. If you don't provide this flag. Compose looks for a file named -`docker-compose.yml` in the working directory. If the file is not found, -Compose looks in each parent directory successively, until it finds the file. +Use the `-f` flag to specify the location of a Compose configuration file. You +can supply multiple `-f` configuration files. When you supply multiple files, +Compose combines them into a single configuration. Compose builds the +configuration in the order you supply the files. Subsequent files override and +add to their successors. -Use a `-` as the filename to read configuration file from stdin. When stdin is -used all paths in the configuration are relative to the current working -directory. +For example, consider this command line: + +``` +$ docker-compose -f docker-compose.yml -f docker-compose.admin.yml run backup_db` +``` + +The `docker-compose.yml` file might specify a `webapp` service. + +``` +webapp: + image: examples/web + ports: + - "8000:8000" + volumes: + - "/data" +``` + +If the `docker-compose.admin.yml` also specifies this same service, any matching +fields will override the previous file. New values, add to the `webapp` service +configuration. + +``` +webapp: + build: . + environment: + - DEBUG=1 +``` + +Use a `-f` with `-` (dash) as the filename to read the configuration from +stdin. When stdin is used all paths in the configuration are +relative to the current working directory. + +The `-f` flag is optional. If you don't provide this flag on the command line, +Compose traverses the working directory and its subdirectories looking for a +`docker-compose.yml` and a `docker-compose.override.yml` file. You must supply +at least the `docker-compose.yml` file. If both files are present, Compose +combines the two files into a single configuration. The configuration in the +`docker-compose.override.yml` file is applied over and in addition to the values +in the `docker-compose.yml` file. + +Each configuration has a project name. If you supply a `-p` flag, you can +specify a project name. If you don't specify the flag, Compose uses the current +directory name. -Each configuration can has a project name. If you supply a `-p` flag, you can specify a project name. If you don't specify the flag, Compose uses the current directory name. ## Where to go next From 577439ea7f6b506e6905ca097abeff7ba82af5e6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 17 Sep 2015 14:28:16 -0400 Subject: [PATCH 196/337] Add a debug log message for config filenames. Signed-off-by: Daniel Nephin --- compose/config/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 3ecdd29d7b..56e6e796ba 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -104,6 +104,8 @@ def find(base_dir, filenames): filenames = [os.path.join(base_dir, f) for f in filenames] else: filenames = get_default_config_files(base_dir) + + log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) From eb20590ca66bb458504be60df7df948835a2eb45 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 29 Jul 2015 16:16:42 +0100 Subject: [PATCH 197/337] Use dockerswarm/dind image instead of doing docker-in-docker ourselves Signed-off-by: Aanand Prasad --- script/dind | 88 -------------------------------------------- script/test-versions | 36 ++++++++++++------ script/wrapdocker | 27 -------------- 3 files changed, 25 insertions(+), 126 deletions(-) delete mode 100755 script/dind delete mode 100755 script/wrapdocker diff --git a/script/dind b/script/dind deleted file mode 100755 index f8fae6379c..0000000000 --- a/script/dind +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -set -e - -# DinD: a wrapper script which allows docker to be run inside a docker container. -# Original version by Jerome Petazzoni -# See the blog post: http://blog.docker.com/2013/09/docker-can-now-run-within-docker/ -# -# This script should be executed inside a docker container in privilieged mode -# ('docker run --privileged', introduced in docker 0.6). - -# Usage: dind CMD [ARG...] - -# apparmor sucks and Docker needs to know that it's in a container (c) @tianon -export container=docker - -# First, make sure that cgroups are mounted correctly. -CGROUP=/cgroup - -mkdir -p "$CGROUP" - -if ! mountpoint -q "$CGROUP"; then - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { - echo >&2 'Could not make a tmpfs mount. Did you use --privileged?' - exit 1 - } -fi - -if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then - mount -t securityfs none /sys/kernel/security || { - echo >&2 'Could not mount /sys/kernel/security.' - echo >&2 'AppArmor detection and -privileged mode might break.' - } -fi - -# Mount the cgroup hierarchies exactly as they are in the parent system. -for SUBSYS in $(cut -d: -f2 /proc/1/cgroup); do - mkdir -p "$CGROUP/$SUBSYS" - if ! mountpoint -q $CGROUP/$SUBSYS; then - mount -n -t cgroup -o "$SUBSYS" cgroup "$CGROUP/$SUBSYS" - fi - - # The two following sections address a bug which manifests itself - # by a cryptic "lxc-start: no ns_cgroup option specified" when - # trying to start containers withina container. - # The bug seems to appear when the cgroup hierarchies are not - # mounted on the exact same directories in the host, and in the - # container. - - # Named, control-less cgroups are mounted with "-o name=foo" - # (and appear as such under /proc//cgroup) but are usually - # mounted on a directory named "foo" (without the "name=" prefix). - # Systemd and OpenRC (and possibly others) both create such a - # cgroup. To avoid the aforementioned bug, we symlink "foo" to - # "name=foo". This shouldn't have any adverse effect. - name="${SUBSYS#name=}" - if [ "$name" != "$SUBSYS" ]; then - ln -s "$SUBSYS" "$CGROUP/$name" - fi - - # Likewise, on at least one system, it has been reported that - # systemd would mount the CPU and CPU accounting controllers - # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" - # but on a directory called "cpu,cpuacct" (note the inversion - # in the order of the groups). This tries to work around it. - if [ "$SUBSYS" = 'cpuacct,cpu' ]; then - ln -s "$SUBSYS" "$CGROUP/cpu,cpuacct" - fi -done - -# Note: as I write those lines, the LXC userland tools cannot setup -# a "sub-container" properly if the "devices" cgroup is not in its -# own hierarchy. Let's detect this and issue a warning. -if ! grep -q :devices: /proc/1/cgroup; then - echo >&2 'WARNING: the "devices" cgroup should be in its own hierarchy.' -fi -if ! grep -qw devices /proc/1/cgroup; then - echo >&2 'WARNING: it looks like the "devices" cgroup is not mounted.' -fi - -# Mount /tmp -mount -t tmpfs none /tmp - -if [ $# -gt 0 ]; then - exec "$@" -fi - -echo >&2 'ERROR: No command specified.' -echo >&2 'You probably want to run hack/make.sh, or maybe a shell?' diff --git a/script/test-versions b/script/test-versions index 88d2554c2b..577cf67e1f 100755 --- a/script/test-versions +++ b/script/test-versions @@ -11,21 +11,35 @@ docker run --rm \ "$TAG" -e pre-commit if [ "$DOCKER_VERSIONS" == "" ]; then - DOCKER_VERSIONS="default" + DOCKER_VERSIONS="$DEFAULT_DOCKER_VERSION" elif [ "$DOCKER_VERSIONS" == "all" ]; then DOCKER_VERSIONS="$ALL_DOCKER_VERSIONS" fi for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" - docker run \ - --rm \ - --privileged \ - --volume="/var/lib/docker" \ - --volume="${COVERAGE_DIR:-$(pwd)/coverage-html}:/code/coverage-html" \ - -e "DOCKER_VERSION=$version" \ - -e "DOCKER_DAEMON_ARGS" \ - --entrypoint="script/dind" \ - "$TAG" \ - script/wrapdocker tox -e py27,py34 -- "$@" + + ( + set -x + + daemon_container_id=$(\ + docker run \ + -d \ + --privileged \ + --volume="/var/lib/docker" \ + --expose="2375" \ + dockerswarm/dind:$version \ + docker -d -H tcp://0.0.0.0:2375 \ + ) + + docker run \ + --rm \ + --link="$daemon_container_id:docker" \ + --env="DOCKER_HOST=tcp://docker:2375" \ + --entrypoint="tox" \ + "$TAG" \ + -e py27,py34 -- "$@" + + docker rm -vf "$daemon_container_id" + ) done diff --git a/script/wrapdocker b/script/wrapdocker deleted file mode 100755 index ab89f5ed64..0000000000 --- a/script/wrapdocker +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -if [ "$DOCKER_VERSION" != "" ] && [ "$DOCKER_VERSION" != "default" ]; then - ln -fs "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" -fi - -# If a pidfile is still around (for example after a container restart), -# delete it so that docker can start. -rm -rf /var/run/docker.pid -docker_command="docker -d $DOCKER_DAEMON_ARGS" ->&2 echo "Starting Docker with: $docker_command" -$docker_command &>/var/log/docker.log & -docker_pid=$! - ->&2 echo "Waiting for Docker to start..." -while ! docker ps &>/dev/null; do - if ! kill -0 "$docker_pid" &>/dev/null; then - >&2 echo "Docker failed to start" - cat /var/log/docker.log - exit 1 - fi - - sleep 1 -done - ->&2 echo ">" "$@" -exec "$@" From 9978c3ea52edbc99f2e293e98c4db5196972d655 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 31 Aug 2015 17:29:25 -0400 Subject: [PATCH 198/337] Update scriptests/test-versions to work with daemon args, and move docker version constants into tests-versions. Signed-off-by: Daniel Nephin --- Dockerfile | 11 ----------- script/test-versions | 45 ++++++++++++++++++++++++-------------------- tox.ini | 1 + 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba508742de..354ba00a4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,17 +66,6 @@ RUN set -ex; \ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 -ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.2-rc1 - -RUN set -ex; \ - curl https://get.docker.com/builds/Linux/x86_64/docker-1.7.1 -o /usr/local/bin/docker-1.7.1; \ - chmod +x /usr/local/bin/docker-1.7.1; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.8.2-rc1 -o /usr/local/bin/docker-1.8.2-rc1; \ - chmod +x /usr/local/bin/docker-1.8.2-rc1 - -# Set the default Docker to be run -RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker - RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/script/test-versions b/script/test-versions index 577cf67e1f..bebc556727 100755 --- a/script/test-versions +++ b/script/test-versions @@ -10,36 +10,41 @@ docker run --rm \ --entrypoint="tox" \ "$TAG" -e pre-commit +ALL_DOCKER_VERSIONS="1.7.1 1.8.2" +DEFAULT_DOCKER_VERSION="1.8.2" + if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$DEFAULT_DOCKER_VERSION" elif [ "$DOCKER_VERSIONS" == "all" ]; then DOCKER_VERSIONS="$ALL_DOCKER_VERSIONS" fi + +BUILD_NUMBER=${BUILD_NUMBER-$USER} + for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" - ( - set -x + daemon_container="compose-dind-$version-$BUILD_NUMBER" + trap "docker rm -vf $daemon_container" EXIT - daemon_container_id=$(\ - docker run \ - -d \ - --privileged \ - --volume="/var/lib/docker" \ - --expose="2375" \ - dockerswarm/dind:$version \ - docker -d -H tcp://0.0.0.0:2375 \ - ) + # TODO: remove when we stop testing against 1.7.x + daemon=$([[ "$version" == "1.7"* ]] && echo "-d" || echo "daemon") - docker run \ - --rm \ - --link="$daemon_container_id:docker" \ - --env="DOCKER_HOST=tcp://docker:2375" \ - --entrypoint="tox" \ - "$TAG" \ - -e py27,py34 -- "$@" + docker run \ + -d \ + --name "$daemon_container" \ + --privileged \ + --volume="/var/lib/docker" \ + dockerswarm/dind:$version \ + docker $daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + + docker run \ + --rm \ + --link="$daemon_container:docker" \ + --env="DOCKER_HOST=tcp://docker:2375" \ + --entrypoint="tox" \ + "$TAG" \ + -e py27,py34 -- "$@" - docker rm -vf "$daemon_container_id" - ) done diff --git a/tox.ini b/tox.ini index 4cb933dd71..901c185173 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = py27,py34,pre-commit usedevelop=True passenv = LD_LIBRARY_PATH + DOCKER_HOST setenv = HOME=/tmp deps = From 8b29a50b525c12e283db3b1aaecc1ab7d6ce6ef3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 16 Sep 2015 16:45:26 -0400 Subject: [PATCH 199/337] Trim the dockerfile and re-use the virtualenv we already have. Signed-off-by: Daniel Nephin --- Dockerfile | 27 +++++++++++++-------------- script/build-linux | 6 +----- script/build-linux-inner | 5 +++-- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 354ba00a4b..c6dbdefd66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,15 +10,16 @@ RUN set -ex; \ zlib1g-dev \ libssl-dev \ git \ - apt-transport-https \ ca-certificates \ curl \ - lxc \ - iptables \ libsqlite3-dev \ ; \ rm -rf /var/lib/apt/lists/* +RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest \ + -o /usr/local/bin/docker && \ + chmod +x /usr/local/bin/docker + # Build Python 2.7.9 from source RUN set -ex; \ curl -LO https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz; \ @@ -69,19 +70,17 @@ ENV LANG en_US.UTF-8 RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ -RUN pip install tox - -ADD requirements.txt /code/ -RUN pip install -r requirements.txt - -ADD requirements-dev.txt /code/ -RUN pip install -r requirements-dev.txt - RUN pip install tox==2.1.1 -ADD . /code/ -RUN pip install --no-deps -e /code +ADD requirements.txt /code/ +ADD requirements-dev.txt /code/ +ADD .pre-commit-config.yaml /code/ +ADD setup.py /code/ +ADD tox.ini /code/ +ADD compose /code/compose/ +RUN tox --notest +ADD . /code/ RUN chown -R user /code/ -ENTRYPOINT ["/usr/local/bin/docker-compose"] +ENTRYPOINT ["/code/.tox/py27/bin/docker-compose"] diff --git a/script/build-linux b/script/build-linux index 4fdf1d926f..bf966fc8ee 100755 --- a/script/build-linux +++ b/script/build-linux @@ -4,8 +4,4 @@ set -ex TAG="docker-compose" docker build -t "$TAG" . -docker run \ - --rm \ - --volume="$(pwd):/code" \ - --entrypoint="script/build-linux-inner" \ - "$TAG" +docker run --rm --entrypoint="script/build-linux-inner" "$TAG" diff --git a/script/build-linux-inner b/script/build-linux-inner index e5d290ebaa..1d0f790504 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -3,11 +3,12 @@ set -ex TARGET=dist/docker-compose-Linux-x86_64 +VENV=/code/.tox/py27 mkdir -p `pwd`/dist chmod 777 `pwd`/dist -pip install -r requirements-build.txt -su -c "pyinstaller docker-compose.spec" user +$VENV/bin/pip install -r requirements-build.txt +su -c "$VENV/bin/pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version From aac916c73ed99e8a2615707e4550a7d21edea135 Mon Sep 17 00:00:00 2001 From: Mike Bailey Date: Fri, 31 Jul 2015 16:35:13 +1000 Subject: [PATCH 200/337] Alphabetise reference list Bring in line with Glossary. https://docs.docker.com/reference/glossary/ Alphabetising list makes makes parsing by humans easier. Signed-off-by: Mike Bailey Conflicts: docs/yml.md --- docs/yml.md | 332 ++++++++++++++++++++++++---------------------------- 1 file changed, 151 insertions(+), 181 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 2f1ae1a6ad..77f76b1083 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -19,22 +19,6 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. -Values for configuration options can contain environment variables, e.g. -`image: postgres:${POSTGRES_VERSION}`. For more details, see the section on -[variable substitution](#variable-substitution). - -### image - -Tag, partial image ID or digest. Can be local or remote - Compose will attempt to -pull if it doesn't exist locally. - - image: ubuntu - image: orchardup/postgresql - image: a4bc65fd - image: busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d - -Using `image` together with either `build` or `dockerfile` is not allowed. Attempting to do so results in an error. - ### build Path to a directory containing a Dockerfile. When the value supplied is a @@ -47,13 +31,17 @@ Compose will build and tag it with a generated name, and use that image thereaft Using `build` together with `image` is not allowed. Attempting to do so results in an error. -### dockerfile +### cap_add, cap_drop -Alternate Dockerfile. +Add or drop container capabilities. +See `man 7 capabilities` for a full list. -Compose will use an alternate file to build with. + cap_add: + - ALL - dockerfile: Dockerfile-alternate + cap_drop: + - NET_ADMIN + - SYS_ADMIN Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. @@ -63,143 +51,49 @@ Override the default command. command: bundle exec thin -p 3000 - -### links +### container_name -Link to containers in another service. Either specify both the service name and -the link alias (`SERVICE:ALIAS`), or just the service name (which will also be -used for the alias). +Specify a custom container name, rather than a generated default name. - links: - - db - - db:database - - redis + container_name: my-web-container -An entry with the alias' name will be created in `/etc/hosts` inside containers -for this service, e.g: +Because Docker container names must be unique, you cannot scale a service +beyond 1 container if you have specified a custom name. Attempting to do so +results in an error. - 172.17.2.186 db - 172.17.2.186 database - 172.17.2.187 redis +### devices -Environment variables will also be created - see the [environment variable -reference](env.md) for details. +List of device mappings. Uses the same format as the `--device` docker +client create option. -### external_links + devices: + - "/dev/ttyUSB0:/dev/ttyUSB0" -Link to containers started outside this `docker-compose.yml` or even outside -of Compose, especially for containers that provide shared or common services. -`external_links` follow semantics similar to `links` when specifying both the -container name and the link alias (`CONTAINER:ALIAS`). +### dns - external_links: - - redis_1 - - project_db_1:mysql - - project_db_1:postgresql +Custom DNS servers. Can be a single value or a list. -### extra_hosts + dns: 8.8.8.8 + dns: + - 8.8.8.8 + - 9.9.9.9 -Add hostname mappings. Use the same values as the docker client `--add-host` parameter. +### dns_search - extra_hosts: - - "somehost:162.242.195.82" - - "otherhost:50.31.209.229" +Custom DNS search domains. Can be a single value or a list. -An entry with the ip address and hostname will be created in `/etc/hosts` inside containers for this service, e.g: + dns_search: example.com + dns_search: + - dc1.example.com + - dc2.example.com - 162.242.195.82 somehost - 50.31.209.229 otherhost +### dockerfile -### ports +Alternate Dockerfile. -Makes an exposed port accessible on a host and the port is available to -any client that can reach that host. Docker binds the exposed port to a random -port on the host within an *ephemeral port range* defined by -`/proc/sys/net/ipv4/ip_local_port_range`. You can also map to a specific port or range of ports. +Compose will use an alternate file to build with. -Acceptable formats for the `ports` value are: - -* `containerPort` -* `ip:hostPort:containerPort` -* `ip::containerPort` -* `hostPort:containerPort` - -You can specify a range for both the `hostPort` and the `containerPort` values. -When specifying ranges, the container port values in the range must match the -number of host port values in the range, for example, -`1234-1236:1234-1236/tcp`. Once a host is running, use the 'docker-compose port' command -to see the actual mapping. - -The following configuration shows examples of the port formats in use: - - ports: - - "3000" - - "3000-3005" - - "8000:8000" - - "9090-9091:8080-8081" - - "49100:22" - - "127.0.0.1:8001:8001" - - "127.0.0.1:5000-5010:5000-5010" - - -When mapping ports, in the `hostPort:containerPort` format, you may -experience erroneous results when using a container port lower than 60. This -happens because YAML parses numbers in the format `xx:yy` as sexagesimal (base -60). To avoid this problem, always explicitly specify your port -mappings as strings. - -### expose - -Expose ports without publishing them to the host machine - they'll only be -accessible to linked services. Only the internal port can be specified. - - expose: - - "3000" - - "8000" - -### volumes - -Mount paths as volumes, optionally specifying a path on the host machine -(`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). - - volumes: - - /var/lib/mysql - - ./cache:/tmp/cache - - ~/configs:/etc/configs/:ro - -You can mount a relative path on the host, which will expand relative to -the directory of the Compose configuration file being used. Relative paths -should always begin with `.` or `..`. - -> Note: No path expansion will be done if you have also specified a -> `volume_driver`. - -### volumes_from - -Mount all of the volumes from another service or container. - - volumes_from: - - service_name - - container_name - -### environment - -Add environment variables. You can use either an array or a dictionary. Any -boolean values; true, false, yes no, need to be enclosed in quotes to ensure -they are not converted to True or False by the YML parser. - -Environment variables with only a key are resolved to their values on the -machine Compose is running on, which can be helpful for secret or host-specific values. - - environment: - RACK_ENV: development - SHOW: 'true' - SESSION_SECRET: - - environment: - - RACK_ENV=development - - SHOW=true - - SESSION_SECRET + dockerfile: Dockerfile-alternate ### env_file @@ -223,6 +117,34 @@ beginning with `#` (i.e. comments) are ignored, as are blank lines. # Set Rails/Rack environment RACK_ENV=development +### environment + +Add environment variables. You can use either an array or a dictionary. Any +boolean values; true, false, yes no, need to be enclosed in quotes to ensure +they are not converted to True or False by the YML parser. + +Environment variables with only a key are resolved to their values on the +machine Compose is running on, which can be helpful for secret or host-specific values. + + environment: + RACK_ENV: development + SHOW: 'true' + SESSION_SECRET: + + environment: + - RACK_ENV=development + - SHOW=true + - SESSION_SECRET + +### expose + +Expose ports without publishing them to the host machine - they'll only be +accessible to linked services. Only the internal port can be specified. + + expose: + - "3000" + - "8000" + ### extends Extend another service, in the current file or another, optionally overriding @@ -267,6 +189,40 @@ service within the current file. For more on `extends`, see the [tutorial](extends.md#example) and [reference](extends.md#reference). +### external_links + +Link to containers started outside this `docker-compose.yml` or even outside +of Compose, especially for containers that provide shared or common services. +`external_links` follow semantics similar to `links` when specifying both the +container name and the link alias (`CONTAINER:ALIAS`). + + external_links: + - redis_1 + - project_db_1:mysql + - project_db_1:postgresql + +### extra_hosts + +Add hostname mappings. Use the same values as the docker client `--add-host` parameter. + + extra_hosts: + - "somehost:162.242.195.82" + - "otherhost:50.31.209.229" + +An entry with the ip address and hostname will be created in `/etc/hosts` inside containers for this service, e.g: + + 162.242.195.82 somehost + 50.31.209.229 otherhost + +### image + +Tag or partial image ID. Can be local or remote - Compose will attempt to +pull if it doesn't exist locally. + + image: ubuntu + image: orchardup/postgresql + image: a4bc65fd + ### labels Add metadata to containers using [Docker labels](http://docs.docker.com/userguide/labels-custom-metadata/). You can use either an array or a dictionary. @@ -283,15 +239,26 @@ It's recommended that you use reverse-DNS notation to prevent your labels from c - "com.example.department=Finance" - "com.example.label-with-empty-value" -### container_name +### links -Specify a custom container name, rather than a generated default name. +Link to containers in another service. Either specify both the service name and +the link alias (`SERVICE:ALIAS`), or just the service name (which will also be +used for the alias). - container_name: my-web-container + links: + - db + - db:database + - redis -Because Docker container names must be unique, you cannot scale a service -beyond 1 container if you have specified a custom name. Attempting to do so -results in an error. +An entry with the alias' name will be created in `/etc/hosts` inside containers +for this service, e.g: + + 172.17.2.186 db + 172.17.2.186 database + 172.17.2.187 redis + +Environment variables will also be created - see the [environment variable +reference](env.md) for details. ### log_driver @@ -336,43 +303,21 @@ container and the host operating system the PID address space. Containers launched with this flag will be able to access and manipulate other containers in the bare-metal machine's namespace and vise-versa. -### dns +### ports -Custom DNS servers. Can be a single value or a list. +Expose ports. Either specify both ports (`HOST:CONTAINER`), or just the container +port (a random host port will be chosen). - dns: 8.8.8.8 - dns: - - 8.8.8.8 - - 9.9.9.9 +> **Note:** When mapping ports in the `HOST:CONTAINER` format, you may experience +> erroneous results when using a container port lower than 60, because YAML will +> parse numbers in the format `xx:yy` as sexagesimal (base 60). For this reason, +> we recommend always explicitly specifying your port mappings as strings. -### cap_add, cap_drop - -Add or drop container capabilities. -See `man 7 capabilities` for a full list. - - cap_add: - - ALL - - cap_drop: - - NET_ADMIN - - SYS_ADMIN - -### dns_search - -Custom DNS search domains. Can be a single value or a list. - - dns_search: example.com - dns_search: - - dc1.example.com - - dc2.example.com - -### devices - -List of device mappings. Uses the same format as the `--device` docker -client create option. - - devices: - - "/dev/ttyUSB0:/dev/ttyUSB0" + ports: + - "3000" + - "8000:8000" + - "49100:22" + - "127.0.0.1:8001:8001" ### security_opt @@ -382,6 +327,31 @@ Override the default labeling scheme for each container. - label:user:USER - label:role:ROLE +### volumes + +Mount paths as volumes, optionally specifying a path on the host machine +(`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). + + volumes: + - /var/lib/mysql + - ./cache:/tmp/cache + - ~/configs:/etc/configs/:ro + +You can mount a relative path on the host, which will expand relative to +the directory of the Compose configuration file being used. Relative paths +should always begin with `.` or `..`. + +> Note: No path expansion will be done if you have also specified a +> `volume_driver`. + +### volumes_from + +Mount all of the volumes from another service or container. + + volumes_from: + - service_name + - container_name + ### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, ipc, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only, volume\_driver Each of these is a single value, analogous to its From 0232fb10d7570ae8efb048e5bbefa0cff5729f29 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 18 Sep 2015 11:30:24 +0100 Subject: [PATCH 201/337] Alphabetise run options Signed-off-by: Mazz Mosley --- docs/yml.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 77f76b1083..81357df3d6 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -352,7 +352,7 @@ Mount all of the volumes from another service or container. - service_name - container_name -### working\_dir, entrypoint, user, hostname, domainname, mac\_address, mem\_limit, memswap\_limit, privileged, ipc, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only, volume\_driver +### cpu\_shares, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, volume\_driver, working\_dir Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. @@ -360,26 +360,24 @@ Each of these is a single value, analogous to its cpu_shares: 73 cpuset: 0,1 - working_dir: /code entrypoint: /code/entrypoint.sh user: postgresql + working_dir: /code - hostname: foo domainname: foo.com - + hostname: foo + ipc: host mac_address: 02:42:ac:11:65:43 mem_limit: 1000000000 memswap_limit: 2000000000 privileged: true - ipc: host - restart: always + read_only: true stdin_open: true tty: true - read_only: true volume_driver: mydriver From db433041b4aa7c1fa8d81c3f5163ec7f87d15c3f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 18 Sep 2015 12:08:09 -0400 Subject: [PATCH 202/337] Restore the dist volume mount for building linux binaries. Signed-off-by: Daniel Nephin --- script/build-linux | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/build-linux b/script/build-linux index 7ce4ffc661..4b8696216d 100755 --- a/script/build-linux +++ b/script/build-linux @@ -6,4 +6,7 @@ set -ex TAG="docker-compose" docker build -t "$TAG" . -docker run --rm --entrypoint="script/build-linux-inner" "$TAG" +docker run \ + --rm --entrypoint="script/build-linux-inner" \ + -v $(pwd)/dist:/code/dist \ + "$TAG" From af7b98ff56517eb7ba0c83f9478f62867a92f078 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 18 Sep 2015 18:12:30 +0200 Subject: [PATCH 203/337] Add bash completion for `docker-compose build --pull` Also adds a fix for an error message on some completions when no compose file is present: docker-compose build awk: cannot open docker-compose.yml (No such file or directory) Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index fe46a334ed..28d94394c2 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -44,7 +44,7 @@ __docker_compose_services_all() { # All services that have an entry with the given key in their compose_file section ___docker_compose_services_with_key() { # flatten sections to one line, then filter lines containing the key and return section name. - awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker_compose_compose_file)}" | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' + awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' } # All services that are defined by a Dockerfile reference @@ -87,7 +87,7 @@ __docker_compose_services_stopped() { _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --no-cache" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build From 006146b2cda8dac0c5b1c79c8eb9008d0a1ada27 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 18 Sep 2015 17:45:35 +0200 Subject: [PATCH 204/337] Add bash completion for `docker-compose run --name` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index fe46a334ed..ae57eaa3e6 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -258,14 +258,14 @@ _docker_compose_run() { compopt -o nospace return ;; - --entrypoint|--user|-u) + --entrypoint|--name|--user|-u) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --no-deps --rm --service-ports --publish -p -T --user -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u" -- "$cur" ) ) ;; *) __docker_compose_services_all From 5509990a7193985b72a22c0af000c5c1185ce4ed Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 18 Sep 2015 14:42:05 +0100 Subject: [PATCH 205/337] Ensure RefResolver works across operating systems Slashes, paths and a tale of woe. Validation now works on windows \o/ Signed-off-by: Mazz Mosley --- compose/config/validation.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 0258c5d94c..959465e987 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,6 +1,7 @@ import json import logging import os +import sys from functools import wraps from docker.utils.ports import split_port @@ -290,12 +291,20 @@ def validate_against_service_schema(config, service_name): def _validate_against_schema(config, schema_filename, format_checker=[], service_name=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) + + if sys.platform == "win32": + file_pre_fix = "///" + config_source_dir = config_source_dir.replace('\\', '/') + else: + file_pre_fix = "//" + + resolver_full_path = "file:{}{}/".format(file_pre_fix, config_source_dir) schema_file = os.path.join(config_source_dir, schema_filename) with open(schema_file, "r") as schema_fh: schema = json.load(schema_fh) - resolver = RefResolver('file://' + config_source_dir + '/', schema) + resolver = RefResolver(resolver_full_path, schema) validation_output = Draft4Validator(schema, resolver=resolver, format_checker=FormatChecker(format_checker)) errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] From 3e4182a48077bb4bd602b7a79622265234d7c518 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 1 Sep 2015 17:40:56 -0700 Subject: [PATCH 206/337] Stub 'run' on Windows Adapted from @dopry's work in https://github.com/docker/compose/pull/1900 Signed-off-by: Aanand Prasad --- compose/cli/main.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9b03ea6763..0fc09efe6c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -8,7 +8,6 @@ import sys from inspect import getdoc from operator import attrgetter -import dockerpty from docker.errors import APIError from requests.exceptions import ReadTimeout @@ -31,6 +30,11 @@ from .log_printer import LogPrinter from .utils import get_version_info from .utils import yesno +WINDOWS = (sys.platform == 'win32') + +if not WINDOWS: + import dockerpty + log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) @@ -335,6 +339,14 @@ class TopLevelCommand(Command): """ service = project.get_service(options['SERVICE']) + detach = options['-d'] + + if WINDOWS and not detach: + raise UserError( + "Interactive mode is not yet supported on Windows.\n" + "Please pass the -d flag when using `docker-compose run`." + ) + if options['--allow-insecure-ssl']: log.warn(INSECURE_SSL_WARNING) @@ -349,7 +361,7 @@ class TopLevelCommand(Command): ) tty = True - if options['-d'] or options['-T'] or not sys.stdin.isatty(): + if detach or options['-T'] or not sys.stdin.isatty(): tty = False if options['COMMAND']: @@ -360,8 +372,8 @@ class TopLevelCommand(Command): container_options = { 'command': command, 'tty': tty, - 'stdin_open': not options['-d'], - 'detach': options['-d'], + 'stdin_open': not detach, + 'detach': detach, } if options['-e']: @@ -407,7 +419,7 @@ class TopLevelCommand(Command): raise e - if options['-d']: + if detach: service.start_container(container) print(container.name) else: From 4ae7f00412e539b9643e83c3eba65718af703f96 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 1 Sep 2015 17:41:09 -0700 Subject: [PATCH 207/337] Build Windows binary Signed-off-by: Aanand Prasad --- script/build-windows.ps1 | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 script/build-windows.ps1 diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 new file mode 100644 index 0000000000..284e44f357 --- /dev/null +++ b/script/build-windows.ps1 @@ -0,0 +1,24 @@ +$ErrorActionPreference = "Stop" +Set-PSDebug -trace 1 + +# Remove virtualenv +if (Test-Path venv) { + Remove-Item -Recurse -Force .\venv +} + +# Remove .pyc files +Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } + +# Create virtualenv +virtualenv .\venv + +# Install dependencies +.\venv\Scripts\easy_install "http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/pywin32-219.win32-py2.7.exe/download" +.\venv\Scripts\pip install -r requirements.txt +.\venv\Scripts\pip install -r requirements-build.txt +.\venv\Scripts\pip install . + +# Build binary +.\venv\Scripts\pyinstaller .\docker-compose.spec +Move-Item -Force .\dist\docker-compose .\dist\docker-compose-Windows-x86_64.exe +.\dist\docker-compose-Windows-x86_64.exe --version From fb304981536722ef42c9688b0688d4be048ccfd1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 18 Sep 2015 17:37:09 +0100 Subject: [PATCH 208/337] Catch WindowsError in call_silently Signed-off-by: Aanand Prasad --- compose/cli/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 0b7ac683d1..26a38af066 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -85,7 +85,12 @@ def call_silently(*args, **kwargs): Like subprocess.call(), but redirects stdout and stderr to /dev/null. """ with open(os.devnull, 'w') as shutup: - return subprocess.call(*args, stdout=shutup, stderr=shutup, **kwargs) + try: + return subprocess.call(*args, stdout=shutup, stderr=shutup, **kwargs) + except WindowsError: + # On Windows, subprocess.call() can still raise exceptions. Normalize + # to POSIXy behaviour by returning a nonzero exit code. + return 1 def is_mac(): From 12b38adfac3b1d1f15addc41e4ecf698007a8717 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 18 Sep 2015 22:42:19 +0200 Subject: [PATCH 209/337] Add zsh completion for 'docker-compose run --name' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 58105dc221..1ff1e7289a 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -250,6 +250,7 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '-d[Detached mode: Run container in the background, print new container name.]' \ + '--name[Assign a name to the container]:name: ' \ '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ From 51ce16cf18812514320a7f3da59d6ee0fff2fb7c Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 18 Sep 2015 22:46:08 +0200 Subject: [PATCH 210/337] Add zsh completion for 'docker-compose build --build' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 58105dc221..912a18287a 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -193,6 +193,7 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '--no-cache[Do not use cache when building the image]' \ + '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; (help) From 22bc174650fddb9b53ece787c489df7dabcef8d9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 17 Sep 2015 15:07:53 -0400 Subject: [PATCH 211/337] Refactor config.load() to remove reduce() and document some types. Signed-off-by: Daniel Nephin --- compose/config/config.py | 49 ++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 56e6e796ba..94c5ab95a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,7 +2,6 @@ import logging import os import sys from collections import namedtuple -from functools import reduce import six import yaml @@ -89,9 +88,22 @@ PATH_START_CHARS = [ log = logging.getLogger(__name__) -ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs') +class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')): + """ + :param working_dir: the directory to use for relative paths in the config + :type working_dir: string + :param config_files: list of configuration files to load + :type config_files: list of :class:`ConfigFile` + """ -ConfigFile = namedtuple('ConfigFile', 'filename config') + +class ConfigFile(namedtuple('_ConfigFile', 'filename config')): + """ + :param filename: filename of the config file + :type filename: string + :param config: contents of the config file + :type config: :class:`dict` + """ def find(base_dir, filenames): @@ -170,10 +182,19 @@ def pre_process_config(config): def load(config_details): - working_dir, configs = config_details + """Load the configuration from a working directory and a list of + configuration files. Files are loaded in order, and merged on top + of each other to create the final configuration. + + Return a fully interpolated, extended and validated configuration. + """ def build_service(filename, service_name, service_dict): - loader = ServiceLoader(working_dir, filename, service_name, service_dict) + loader = ServiceLoader( + config_details.working_dir, + filename, + service_name, + service_dict) service_dict = loader.make_service_dict() validate_paths(service_dict) return service_dict @@ -187,21 +208,19 @@ def load(config_details): ] def merge_services(base, override): + all_service_names = set(base) | set(override) return { name: merge_service_dicts(base.get(name, {}), override.get(name, {})) - for name in set(base) | set(override) + for name in all_service_names } - def combine_configs(base, override): - service_dicts = load_file(base.filename, base.config) - if not override: - return service_dicts + config_file = config_details.config_files[0] + for next_file in config_details.config_files[1:]: + config_file = ConfigFile( + config_file.filename, + merge_services(config_file.config, next_file.config)) - return ConfigFile( - override.filename, - merge_services(base.config, override.config)) - - return reduce(combine_configs, configs + [None]) + return load_file(config_file.filename, config_file.config) class ServiceLoader(object): From c9083e21c81576ba7b8f27dfd952f269cc25a7fd Mon Sep 17 00:00:00 2001 From: Vojta Orgon Date: Mon, 21 Sep 2015 11:59:23 +0200 Subject: [PATCH 212/337] Flag to skip all pull errors when pulling images. Signed-off-by: Vojta Orgon --- compose/cli/main.py | 2 ++ compose/project.py | 4 ++-- compose/service.py | 11 +++++++++-- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + docs/reference/pull.md | 3 +++ .../simple-composefile/ignore-pull-failures.yml | 6 ++++++ tests/integration/cli_test.py | 7 +++++++ 8 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/simple-composefile/ignore-pull-failures.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 9b03ea6763..f32b2a5292 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -270,6 +270,7 @@ class TopLevelCommand(Command): Usage: pull [options] [SERVICE...] Options: + --ignore-pull-failures Pull what it can and ignores images with pull failures. --allow-insecure-ssl Deprecated - no effect. """ if options['--allow-insecure-ssl']: @@ -277,6 +278,7 @@ class TopLevelCommand(Command): project.pull( service_names=options['SERVICE'], + ignore_pull_failures=options.get('--ignore-pull-failures') ) def rm(self, project, options): diff --git a/compose/project.py b/compose/project.py index f34cc0c349..4750a7a9ae 100644 --- a/compose/project.py +++ b/compose/project.py @@ -311,9 +311,9 @@ class Project(object): return plans - def pull(self, service_names=None): + def pull(self, service_names=None, ignore_pull_failures=False): for service in self.get_services(service_names, include_deps=True): - service.pull() + service.pull(ignore_pull_failures) def containers(self, service_names=None, stopped=False, one_off=False): if service_names: diff --git a/compose/service.py b/compose/service.py index cf3b627091..960d3936bf 100644 --- a/compose/service.py +++ b/compose/service.py @@ -769,7 +769,7 @@ class Service(object): return True return False - def pull(self): + def pull(self, ignore_pull_failures=False): if 'image' not in self.options: return @@ -781,7 +781,14 @@ class Service(object): tag=tag, stream=True, ) - stream_output(output, sys.stdout) + + try: + stream_output(output, sys.stdout) + except StreamOutputError as e: + if not ignore_pull_failures: + raise + else: + log.error(six.text_type(e)) class Net(object): diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 28d94394c2..ff09205cb7 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -212,7 +212,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures" -- "$cur" ) ) ;; *) __docker_compose_services_from_image diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 58105dc221..99cb4dc57f 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -237,6 +237,7 @@ __docker-compose_subcommand() { (pull) _arguments \ $opts_help \ + '--ignore-pull-failures[Pull what it can and ignores images with pull failures.]' \ '*:services:__docker-compose_services_from_image' && ret=0 ;; (rm) diff --git a/docs/reference/pull.md b/docs/reference/pull.md index d655dd93be..5ec184b72c 100644 --- a/docs/reference/pull.md +++ b/docs/reference/pull.md @@ -13,6 +13,9 @@ parent = "smn_compose_cli" ``` Usage: pull [options] [SERVICE...] + +Options: +--ignore-pull-failures Pull what it can and ignores images with pull failures. ``` Pulls service images. diff --git a/tests/fixtures/simple-composefile/ignore-pull-failures.yml b/tests/fixtures/simple-composefile/ignore-pull-failures.yml new file mode 100644 index 0000000000..a28f792233 --- /dev/null +++ b/tests/fixtures/simple-composefile/ignore-pull-failures.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: top +another: + image: nonexisting-image:latest + command: top diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 9dadd0368d..56e65a6dd2 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -97,6 +97,13 @@ class CLITestCase(DockerClientTestCase): 'Pulling digest (busybox@' 'sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') + @mock.patch('compose.service.log') + def test_pull_with_ignore_pull_failures(self, mock_logging): + self.command.dispatch(['-f', 'ignore-pull-failures.yml', 'pull', '--ignore-pull-failures'], None) + mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') + mock_logging.info.assert_any_call('Pulling another (nonexisting-image:latest)...') + mock_logging.error.assert_any_call('Error: image library/nonexisting-image:latest not found') + @mock.patch('sys.stdout', new_callable=StringIO) def test_build_plain(self, mock_stdout): self.command.base_dir = 'tests/fixtures/simple-dockerfile' From e5eaf68490098cb89cf9d6ad8b4eaa96bafd0450 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 21 Jul 2015 15:33:45 +0100 Subject: [PATCH 213/337] Remove custom docker client initialization logic Signed-off-by: Aanand Prasad --- compose/cli/docker_client.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 601b0b9aab..2c634f3376 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -1,9 +1,8 @@ import logging import os -import ssl from docker import Client -from docker import tls +from docker.utils import kwargs_from_env from ..const import HTTP_TIMEOUT @@ -15,31 +14,10 @@ def docker_client(): Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ - cert_path = os.environ.get('DOCKER_CERT_PATH', '') - if cert_path == '': - cert_path = os.path.join(os.environ.get('HOME', ''), '.docker') - - base_url = os.environ.get('DOCKER_HOST') - api_version = os.environ.get('COMPOSE_API_VERSION', '1.19') - - tls_config = None - - if os.environ.get('DOCKER_TLS_VERIFY', '') != '': - parts = base_url.split('://', 1) - base_url = '%s://%s' % ('https', parts[1]) - - client_cert = (os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')) - ca_cert = os.path.join(cert_path, 'ca.pem') - - tls_config = tls.TLSConfig( - ssl_version=ssl.PROTOCOL_TLSv1, - verify=True, - assert_hostname=False, - client_cert=client_cert, - ca_cert=ca_cert, - ) - if 'DOCKER_CLIENT_TIMEOUT' in os.environ: log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') - return Client(base_url=base_url, tls=tls_config, version=api_version, timeout=HTTP_TIMEOUT) + kwargs = kwargs_from_env(assert_hostname=False) + kwargs['version'] = os.environ.get('COMPOSE_API_VERSION', '1.19') + kwargs['timeout'] = HTTP_TIMEOUT + return Client(**kwargs) From 62ca8469b08250329fee613da704f0186dcdd488 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Mon, 21 Sep 2015 08:57:01 -0700 Subject: [PATCH 214/337] Fixing misspelling of WordPress Signed-off-by: Mary Anthony --- docs/index.md | 4 ++-- docs/reference/overview.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 21a8610e64..67a6802b06 100644 --- a/docs/index.md +++ b/docs/index.md @@ -196,7 +196,7 @@ your services once you've finished with them: At this point, you have seen the basics of how Compose works. - Next, try the quick start guide for [Django](django.md), - [Rails](rails.md), or [Wordpress](wordpress.md). + [Rails](rails.md), or [WordPress](wordpress.md). - See the reference guides for complete details on the [commands](/reference), the [configuration file](yml.md) and [environment variables](env.md). @@ -224,7 +224,7 @@ For more information and resources, please visit the [Getting Help project page] - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 002607118d..9f08246e09 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -83,7 +83,7 @@ it failed. Defaults to 60 seconds. - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with Wordpress](wordpress.md) +- [Get started with WordPress](wordpress.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) From c37a0c38a2c4c9c49c5e591427d382c8a046635d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 12:24:56 -0400 Subject: [PATCH 215/337] Fix a test case that assumes busybox image id. Signed-off-by: Daniel Nephin --- tests/integration/service_test.py | 7 +++++-- tests/integration/testcases.py | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 040098c9e7..2c9c6fc204 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,6 +12,7 @@ from six import text_type from .. import mock from .testcases import DockerClientTestCase +from .testcases import pull_busybox from compose import __version__ from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF @@ -549,8 +550,10 @@ class ServiceTest(DockerClientTestCase): }) def test_create_with_image_id(self): - # Image id for the current busybox:latest - service = self.create_service('foo', image='8c2e06607696') + # Get image id for the current busybox:latest + pull_busybox(self.client) + image_id = self.client.inspect_image('busybox:latest')['Id'][:12] + service = self.create_service('foo', image=image_id) service.create_container() def test_scale(self): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 4557c07b6a..26a0a108a1 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +from docker import errors + from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import ServiceLoader @@ -9,6 +11,13 @@ from compose.progress_stream import stream_output from compose.service import Service +def pull_busybox(client): + try: + client.inspect_image('busybox:latest') + except errors.APIError: + client.pull('busybox:latest', stream=False) + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): From 0058d5dd4f03317af0bacfb9e2d2e045b4bb1da0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 11:19:35 -0400 Subject: [PATCH 216/337] Add appveyor config Signed-off-by: Daniel Nephin --- appveyor.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000000..639591e9cf --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,15 @@ + +install: + - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" + - "python --version" + - "pip install tox==2.1.1" + + +# Build the binary after tests +build: false + +test_script: + - "tox -e py27,py34 -- tests/unit" + +after_test: + - ps: ".\\script\\build-windows.ps1" From d5991761cd678dc48f2924947056fe40415592d2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 14:31:55 -0400 Subject: [PATCH 217/337] Fix building the binary on appveyor, and have it create an artifact. Signed-off-by: Daniel Nephin --- appveyor.yml | 9 +++++++-- script/build-osx | 2 +- script/build-windows.ps1 | 15 ++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 639591e9cf..acf8bff34a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,9 +1,10 @@ +version: '{branch}-{build}' + install: - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.1.1" - + - "pip install tox==2.1.1 virtualenv==13.1.2" # Build the binary after tests build: false @@ -13,3 +14,7 @@ test_script: after_test: - ps: ".\\script\\build-windows.ps1" + +artifacts: + - path: .\dist\docker-compose-Windows-x86_64.exe + name: "Compose Windows binary" diff --git a/script/build-osx b/script/build-osx index 11b6ecc694..15a7bbc541 100755 --- a/script/build-osx +++ b/script/build-osx @@ -9,7 +9,7 @@ rm -rf venv virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt -venv/bin/pip install . +venv/bin/pip install --no-deps . venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 284e44f357..63be086524 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -13,12 +13,21 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -.\venv\Scripts\easy_install "http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/pywin32-219.win32-py2.7.exe/download" +.\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt -.\venv\Scripts\pip install -r requirements-build.txt -.\venv\Scripts\pip install . +.\venv\Scripts\pip install --no-deps . + +# TODO: pip warns when installing from a git sha, so we need to set ErrorAction to +# 'Continue'. See +# https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 +# This can be removed once pyinstaller 3.x is released and we upgrade +$ErrorActionPreference = "Continue" +.\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt # Build binary +# pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue .\venv\Scripts\pyinstaller .\docker-compose.spec +$ErrorActionPreference = "Stop" + Move-Item -Force .\dist\docker-compose .\dist\docker-compose-Windows-x86_64.exe .\dist\docker-compose-Windows-x86_64.exe --version From 78c0734cbd3f553af628a5c19843dbc523cdf28f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 17:37:14 -0400 Subject: [PATCH 218/337] Disable some tests in windows for now. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 6 +++--- compose/const.py | 2 ++ tests/unit/cli_test.py | 3 +++ tests/unit/config/config_test.py | 10 ++++++++++ tests/unit/service_test.py | 3 +++ 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index bb12b62c28..60e60b795d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -16,6 +16,7 @@ from .. import legacy from ..config import parse_environment from ..const import DEFAULT_TIMEOUT from ..const import HTTP_TIMEOUT +from ..const import IS_WINDOWS_PLATFORM from ..progress_stream import StreamOutputError from ..project import ConfigurationError from ..project import NoSuchService @@ -30,9 +31,8 @@ from .log_printer import LogPrinter from .utils import get_version_info from .utils import yesno -WINDOWS = (sys.platform == 'win32') -if not WINDOWS: +if not IS_WINDOWS_PLATFORM: import dockerpty log = logging.getLogger(__name__) @@ -343,7 +343,7 @@ class TopLevelCommand(Command): detach = options['-d'] - if WINDOWS and not detach: + if IS_WINDOWS_PLATFORM and not detach: raise UserError( "Interactive mode is not yet supported on Windows.\n" "Please pass the -d flag when using `docker-compose run`." diff --git a/compose/const.py b/compose/const.py index dbfa56b8cd..b43e655b19 100644 --- a/compose/const.py +++ b/compose/const.py @@ -1,4 +1,5 @@ import os +import sys DEFAULT_TIMEOUT = 10 LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' @@ -8,3 +9,4 @@ LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) +IS_WINDOWS_PLATFORM = (sys.platform == 'win32') diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 321df97a53..0c78e6bbfe 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -5,6 +5,7 @@ import os import docker import py +import pytest from .. import mock from .. import unittest @@ -13,6 +14,7 @@ from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand +from compose.const import IS_WINDOWS_PLATFORM from compose.service import Service @@ -81,6 +83,7 @@ class CLITestCase(unittest.TestCase): with self.assertRaises(NoSuchCommand): TopLevelCommand().dispatch(['help', 'nonexistent'], None) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") @mock.patch('compose.cli.main.dockerpty', autospec=True) def test_run_with_environment_merged_with_options_list(self, mock_dockerpty): command = TopLevelCommand() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 79864ec784..2dfa764df2 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5,8 +5,11 @@ import shutil import tempfile from operator import itemgetter +import pytest + from compose.config import config from compose.config.errors import ConfigurationError +from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest @@ -92,6 +95,7 @@ class ConfigTest(unittest.TestCase): ) ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_load_with_multiple_files(self): base_file = config.ConfigFile( 'base.yaml', @@ -410,6 +414,7 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' @@ -422,6 +427,7 @@ class InterpolationTest(unittest.TestCase): )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): os.environ['HOME'] = '/home/user' @@ -817,6 +823,7 @@ class EnvTest(unittest.TestCase): {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_resolve_path(self): os.environ['HOSTENV'] = '/tmp' @@ -1073,6 +1080,7 @@ class ExtendsTest(unittest.TestCase): for service in service_dicts: self.assertTrue(service['hostname'], expected_interpolated_value) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_volume_path(self): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') @@ -1108,6 +1116,7 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(dicts[0]['environment'], {'FOO': '1'}) +@pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): working_dir = '/home/user/somedir' @@ -1129,6 +1138,7 @@ class ExpandPathTest(unittest.TestCase): self.assertEqual(result, user_path + 'otherdir/somefile') +@pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class BuildPathTest(unittest.TestCase): def setUp(self): self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5f7ae94875..a1c195acf0 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,9 +2,11 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker +import pytest from .. import mock from .. import unittest +from compose.const import IS_WINDOWS_PLATFORM from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT @@ -439,6 +441,7 @@ def mock_get_image(images): raise NoSuchImageError() +@pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ServiceVolumesTest(unittest.TestCase): def setUp(self): From bd1373f52773be08f32d89e7ac207bbae89e0e66 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 10:31:42 -0400 Subject: [PATCH 219/337] Bump 1.4.2 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 6 ++++++ docs/install.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a054a0aef4..598f5e5794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ Change log ========== +1.4.2 (2015-09-22) +------------------ + +- Fixed a regression in the 1.4.1 release that would cause `docker-compose up` + without the `-d` option to exit immediately. + 1.4.1 (2015-09-10) ------------------ diff --git a/docs/install.md b/docs/install.md index 517b2901bb..bc1f8f78c1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -52,7 +52,7 @@ To install Compose, do the following: 6. Test the installation. $ docker-compose --version - docker-compose version: 1.4.1 + docker-compose version: 1.4.2 ## Upgrading From 91b227d133a33e2dd0b8cc06bba33bf389ff7e3f Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Wed, 9 Sep 2015 22:30:36 +0100 Subject: [PATCH 220/337] Allow to extend service using shorthand notation. Closes #1989 Signed-off-by: Karol Duleba --- compose/config/config.py | 10 ++++++--- compose/config/fields_schema.json | 21 ++++++++++++------- .../extends/verbose-and-shorthand.yml | 15 +++++++++++++ tests/unit/config/config_test.py | 20 ++++++++++++++++++ 4 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/extends/verbose-and-shorthand.yml diff --git a/compose/config/config.py b/compose/config/config.py index 94c5ab95a6..55c717f421 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -275,15 +275,19 @@ class ServiceLoader(object): self.service_dict['environment'] = env def validate_and_construct_extends(self): + extends = self.service_dict['extends'] + if not isinstance(extends, dict): + extends = {'service': extends} + validate_extends_file_path( self.service_name, - self.service_dict['extends'], + extends, self.filename ) self.extended_config_path = self.get_extended_config_path( - self.service_dict['extends'] + extends ) - self.extended_service_name = self.service_dict['extends']['service'] + self.extended_service_name = extends['service'] full_extended_config = pre_process_config( load_yaml(self.extended_config_path) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 6fce299cbb..07b17cb22f 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -57,14 +57,21 @@ }, "extends": { - "type": "object", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", - "properties": { - "service": {"type": "string"}, - "file": {"type": "string"} - }, - "required": ["service"], - "additionalProperties": false + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] }, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, diff --git a/tests/fixtures/extends/verbose-and-shorthand.yml b/tests/fixtures/extends/verbose-and-shorthand.yml new file mode 100644 index 0000000000..d381630275 --- /dev/null +++ b/tests/fixtures/extends/verbose-and-shorthand.yml @@ -0,0 +1,15 @@ +base: + image: busybox + environment: + - "BAR=1" + +verbose: + extends: + service: base + environment: + - "FOO=1" + +shorthand: + extends: base + environment: + - "FOO=2" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2dfa764df2..2c3c5a3a12 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1115,6 +1115,26 @@ class ExtendsTest(unittest.TestCase): dicts = load_from_filename('tests/fixtures/extends/valid-common-config.yml') self.assertEqual(dicts[0]['environment'], {'FOO': '1'}) + def test_extended_service_with_verbose_and_shorthand_way(self): + services = load_from_filename('tests/fixtures/extends/verbose-and-shorthand.yml') + self.assertEqual(service_sort(services), service_sort([ + { + 'name': 'base', + 'image': 'busybox', + 'environment': {'BAR': '1'}, + }, + { + 'name': 'verbose', + 'image': 'busybox', + 'environment': {'BAR': '1', 'FOO': '1'}, + }, + { + 'name': 'shorthand', + 'image': 'busybox', + 'environment': {'BAR': '1', 'FOO': '2'}, + }, + ])) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From bb470798d401cb6c991c4d51a0a14915b986b825 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 1 Oct 2015 12:26:55 +0100 Subject: [PATCH 221/337] Pass all DOCKER_ env vars to py.test This ensures that `tox` will run against SSL-protected Docker daemons. Signed-off-by: Aanand Prasad --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 901c185173..dbf639201b 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ usedevelop=True passenv = LD_LIBRARY_PATH DOCKER_HOST + DOCKER_CERT_PATH + DOCKER_TLS_VERIFY setenv = HOME=/tmp deps = From e38334efbdbac3a5e0a652d4771c2e51e1d73810 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 1 Oct 2015 12:22:59 +0100 Subject: [PATCH 222/337] Don't expand volume names Only expand volume host paths if they begin with a dot. This is a breaking change. The deprecation warning preparing users for this change has been removed. Signed-off-by: Aanand Prasad --- compose/config/config.py | 21 ++------- tests/unit/config/config_test.py | 76 +++++++++++++------------------- 2 files changed, 34 insertions(+), 63 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 94c5ab95a6..0444ba3a6b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -78,13 +78,6 @@ SUPPORTED_FILENAMES = [ DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' -PATH_START_CHARS = [ - '/', - '.', - '~', -] - - log = logging.getLogger(__name__) @@ -495,18 +488,10 @@ def resolve_volume_path(volume, working_dir, service_name): container_path = os.path.expanduser(container_path) if host_path is not None: - if not any(host_path.startswith(c) for c in PATH_START_CHARS): - log.warn( - 'Warning: the mapping "{0}:{1}" in the volumes config for ' - 'service "{2}" is ambiguous. In a future version of Docker, ' - 'it will designate a "named" volume ' - '(see https://github.com/docker/docker/pull/14242). ' - 'To prevent unexpected behaviour, change it to "./{0}:{1}"' - .format(host_path, container_path, service_name) - ) - + if host_path.startswith('.'): + host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) - return "%s:%s" % (expand_path(working_dir, host_path), container_path) + return "{}:{}".format(host_path, container_path) else: return container_path diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2dfa764df2..3269cdff87 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -414,6 +414,12 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) + +class VolumeConfigTest(unittest.TestCase): + def test_no_binding(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['/data']) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): @@ -434,59 +440,39 @@ class InterpolationTest(unittest.TestCase): d = make_service_dict('foo', {'build': '.', 'volumes': ['~:/container/path']}, working_dir='.') self.assertEqual(d['volumes'], ['/home/user:/container/path']) - @mock.patch.dict(os.environ) - def test_volume_binding_with_local_dir_name_raises_warning(self): - def make_dict(**config): - config['build'] = '.' - make_service_dict('foo', config, working_dir='.') + def test_name_does_not_expand(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['mydatavolume:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['mydatavolume:/data']) - with mock.patch('compose.config.config.log.warn') as warn: - make_dict(volumes=['/container/path']) - self.assertEqual(0, warn.call_count) + def test_absolute_posix_path_does_not_expand(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['/var/lib/data:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['/var/lib/data:/data']) - make_dict(volumes=['/data:/container/path']) - self.assertEqual(0, warn.call_count) + def test_absolute_windows_path_does_not_expand(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['C:\\data:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['C:\\data:/data']) - make_dict(volumes=['.:/container/path']) - self.assertEqual(0, warn.call_count) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') + def test_relative_path_does_expand_posix(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='/home/me/myproject') + self.assertEqual(d['volumes'], ['/home/me/myproject/data:/data']) - make_dict(volumes=['..:/container/path']) - self.assertEqual(0, warn.call_count) + d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='/home/me/myproject') + self.assertEqual(d['volumes'], ['/home/me/myproject:/data']) - make_dict(volumes=['./data:/container/path']) - self.assertEqual(0, warn.call_count) + d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='/home/me/myproject') + self.assertEqual(d['volumes'], ['/home/me/otherproject:/data']) - make_dict(volumes=['../data:/container/path']) - self.assertEqual(0, warn.call_count) + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') + def test_relative_path_does_expand_windows(self): + d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) - make_dict(volumes=['.profile:/container/path']) - self.assertEqual(0, warn.call_count) + d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='C:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject:/data']) - make_dict(volumes=['~:/container/path']) - self.assertEqual(0, warn.call_count) - - make_dict(volumes=['~/data:/container/path']) - self.assertEqual(0, warn.call_count) - - make_dict(volumes=['~tmp:/container/path']) - self.assertEqual(0, warn.call_count) - - make_dict(volumes=['data:/container/path'], volume_driver='mydriver') - self.assertEqual(0, warn.call_count) - - make_dict(volumes=['data:/container/path']) - self.assertEqual(1, warn.call_count) - warning = warn.call_args[0][0] - self.assertIn('"data:/container/path"', warning) - self.assertIn('"./data:/container/path"', warning) - - def test_named_volume_with_driver_does_not_expand(self): - d = make_service_dict('foo', { - 'build': '.', - 'volumes': ['namedvolume:/data'], - 'volume_driver': 'foodriver', - }, working_dir='.') - self.assertEqual(d['volumes'], ['namedvolume:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='C:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['C:\\Users\\me\\otherproject:/data']) @mock.patch.dict(os.environ) def test_home_directory_with_driver_does_not_expand(self): From f4cd5b1d45f0eee1a731af1664c75d27e9b4aa18 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 22 Sep 2015 16:13:42 +0100 Subject: [PATCH 223/337] Handle windows volume paths When a relative path is expanded and we're on a windows platform, it expands to include the drive, eg C:\ , which was causing a ConfigError as we split on ":" in parse_volume_spec and that was giving too many parts. Use os.path.splitdrive instead of manually calculating the drive. This should help us deal with windows drives as part of the volume path better than us doing it manually. Signed-off-by: Mazz Mosley --- compose/config/config.py | 11 ++++++----- compose/const.py | 1 + compose/service.py | 14 ++++++++++++++ tests/unit/config/config_test.py | 15 +++++++++++++++ tests/unit/service_test.py | 15 +++++++++++++++ 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0444ba3a6b..9e9cb857fb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -526,12 +526,13 @@ def path_mappings_from_dict(d): return [join_path_mapping(v) for v in d.items()] -def split_path_mapping(string): - if ':' in string: - (host, container) = string.split(':', 1) - return (container, host) +def split_path_mapping(volume_path): + drive, volume_config = os.path.splitdrive(volume_path) + if ':' in volume_config: + (host, container) = volume_config.split(':', 1) + return (container, drive + host) else: - return (string, None) + return (volume_path, None) def join_path_mapping(pair): diff --git a/compose/const.py b/compose/const.py index b43e655b19..b04b7e7e72 100644 --- a/compose/const.py +++ b/compose/const.py @@ -2,6 +2,7 @@ import os import sys DEFAULT_TIMEOUT = 10 +IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' diff --git a/compose/service.py b/compose/service.py index 960d3936bf..4df10fbb10 100644 --- a/compose/service.py +++ b/compose/service.py @@ -20,6 +20,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.validation import VALID_NAME_CHARS from .const import DEFAULT_TIMEOUT +from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -937,7 +938,20 @@ def build_volume_binding(volume_spec): def parse_volume_spec(volume_config): + """ + A volume_config string, which is a path, split it into external:internal[:mode] + parts to be returned as a valid VolumeSpec tuple. + """ parts = volume_config.split(':') + + if IS_WINDOWS_PLATFORM: + # relative paths in windows expand to include the drive, eg C:\ + # so we join the first 2 parts back together to count as one + drive, volume_path = os.path.splitdrive(volume_config) + windows_parts = volume_path.split(":") + windows_parts[0] = os.path.join(drive, windows_parts[0]) + parts = windows_parts + if len(parts) > 3: raise ConfigError("Volume %s has incorrect format, should be " "external:internal[:mode]" % volume_config) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3269cdff87..cf299738f1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1124,6 +1124,21 @@ class ExpandPathTest(unittest.TestCase): self.assertEqual(result, user_path + 'otherdir/somefile') +class VolumePathTest(unittest.TestCase): + + @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') + def test_split_path_mapping_with_windows_path(self): + windows_volume_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config:/opt/connect/config:ro" + expected_mapping = ( + "/opt/connect/config:ro", + "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" + ) + + mapping = config.split_path_mapping(windows_volume_path) + + self.assertEqual(mapping, expected_mapping) + + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class BuildPathTest(unittest.TestCase): def setUp(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a1c195acf0..b0cee1ee16 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -466,6 +466,21 @@ class ServiceVolumesTest(unittest.TestCase): with self.assertRaises(ConfigError): parse_volume_spec('one:two:three:four') + @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') + def test_parse_volume_windows_relative_path(self): + windows_relative_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config:\\opt\\connect\\config:ro" + + spec = parse_volume_spec(windows_relative_path) + + self.assertEqual( + spec, + ( + "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config", + "\\opt\\connect\\config", + "ro" + ) + ) + def test_build_volume_binding(self): binding = build_volume_binding(parse_volume_spec('/outside:/inside')) self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) From 58270d88592e6a097763ce0052ef6a8d22e9bbcb Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 23 Sep 2015 17:08:41 +0100 Subject: [PATCH 224/337] Remove duplicate and re-order alphabetically Signed-off-by: Mazz Mosley --- compose/const.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compose/const.py b/compose/const.py index b04b7e7e72..1b6894189e 100644 --- a/compose/const.py +++ b/compose/const.py @@ -2,6 +2,7 @@ import os import sys DEFAULT_TIMEOUT = 10 +HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' @@ -9,5 +10,3 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' -HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) -IS_WINDOWS_PLATFORM = (sys.platform == 'win32') From 2276326d7ecc4b3bbc30d1acaaa0213669b7ad59 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 1 Oct 2015 11:06:15 +0100 Subject: [PATCH 225/337] volume path compatibility with engine An expanded windows path of c:\shiny\thing:\shiny:rw needs to be changed to be linux style path, including the drive, like /c/shiny/thing /shiny to be mounted successfully by the engine. Signed-off-by: Mazz Mosley --- compose/service.py | 47 +++++++++++++++++++++++--------- tests/unit/config/config_test.py | 3 +- tests/unit/service_test.py | 7 ++--- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/compose/service.py b/compose/service.py index 4df10fbb10..c9ca00ae41 100644 --- a/compose/service.py +++ b/compose/service.py @@ -937,33 +937,54 @@ def build_volume_binding(volume_spec): return volume_spec.internal, "{}:{}:{}".format(*volume_spec) +def normalize_paths_for_engine(external_path, internal_path): + """ + Windows paths, c:\my\path\shiny, need to be changed to be compatible with + the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ + """ + if IS_WINDOWS_PLATFORM: + if external_path: + drive, tail = os.path.splitdrive(external_path) + + if drive: + reformatted_drive = "/{}".format(drive.replace(":", "")) + external_path = reformatted_drive + tail + + external_path = "/".join(external_path.split("\\")) + + return external_path, "/".join(internal_path.split("\\")) + else: + return external_path, internal_path + + def parse_volume_spec(volume_config): """ - A volume_config string, which is a path, split it into external:internal[:mode] - parts to be returned as a valid VolumeSpec tuple. + Parse a volume_config path and split it into external:internal[:mode] + parts to be returned as a valid VolumeSpec. """ - parts = volume_config.split(':') - if IS_WINDOWS_PLATFORM: # relative paths in windows expand to include the drive, eg C:\ # so we join the first 2 parts back together to count as one - drive, volume_path = os.path.splitdrive(volume_config) - windows_parts = volume_path.split(":") - windows_parts[0] = os.path.join(drive, windows_parts[0]) - parts = windows_parts + drive, tail = os.path.splitdrive(volume_config) + parts = tail.split(":") + + if drive: + parts[0] = drive + parts[0] + else: + parts = volume_config.split(':') if len(parts) > 3: raise ConfigError("Volume %s has incorrect format, should be " "external:internal[:mode]" % volume_config) if len(parts) == 1: - external = None - internal = os.path.normpath(parts[0]) + external, internal = normalize_paths_for_engine(None, os.path.normpath(parts[0])) else: - external = os.path.normpath(parts[0]) - internal = os.path.normpath(parts[1]) + external, internal = normalize_paths_for_engine(os.path.normpath(parts[0]), os.path.normpath(parts[1])) - mode = parts[2] if len(parts) == 3 else 'rw' + mode = 'rw' + if len(parts) == 3: + mode = parts[2] return VolumeSpec(external, internal, mode) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index cf299738f1..c06a2a3279 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -420,7 +420,6 @@ class VolumeConfigTest(unittest.TestCase): d = make_service_dict('foo', {'build': '.', 'volumes': ['/data']}, working_dir='.') self.assertEqual(d['volumes'], ['/data']) - @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' @@ -433,7 +432,7 @@ class VolumeConfigTest(unittest.TestCase): )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) - @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): os.environ['HOME'] = '/home/user' diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b0cee1ee16..a3db0d4343 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -441,7 +441,6 @@ def mock_get_image(images): raise NoSuchImageError() -@pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ServiceVolumesTest(unittest.TestCase): def setUp(self): @@ -468,15 +467,15 @@ class ServiceVolumesTest(unittest.TestCase): @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') def test_parse_volume_windows_relative_path(self): - windows_relative_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config:\\opt\\connect\\config:ro" + windows_relative_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" spec = parse_volume_spec(windows_relative_path) self.assertEqual( spec, ( - "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config", - "\\opt\\connect\\config", + "/c/Users/me/Documents/shiny/config", + "/opt/shiny/config", "ro" ) ) From af8032a5f4a5075d71c220bfafbadcbbebbcb5b7 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 1 Oct 2015 12:09:32 +0100 Subject: [PATCH 226/337] Fix incorrect test name I'd written relative, when I meant absolute. D'oh. Signed-off-by: Mazz Mosley --- tests/unit/service_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a3db0d4343..c682b82377 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -466,10 +466,10 @@ class ServiceVolumesTest(unittest.TestCase): parse_volume_spec('one:two:three:four') @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') - def test_parse_volume_windows_relative_path(self): - windows_relative_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" + def test_parse_volume_windows_absolute_path(self): + windows_absolute_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" - spec = parse_volume_spec(windows_relative_path) + spec = parse_volume_spec(windows_absolute_path) self.assertEqual( spec, From bef5c2238e7cefa12cb34b814dc536fa46e9773f Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 2 Oct 2015 15:29:26 +0100 Subject: [PATCH 227/337] Skip a test for now This needs resolving outside of this PR, as it is a much bigger piece of work. https://github.com/docker/compose/issues/2128 Signed-off-by: Mazz Mosley --- tests/unit/config/config_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c06a2a3279..b505740f57 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -463,6 +463,7 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/home/me/otherproject:/data']) @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='waiting for this to be resolved: https://github.com/docker/compose/issues/2128') def test_relative_path_does_expand_windows(self): d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) From da91b81bb89907e5bffa6553540b248d0802a3ca Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 01:44:25 -0400 Subject: [PATCH 228/337] Add scripts for automating parts of the release. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 57 +++++------------- script/release/cherry-pick-pr | 28 +++++++++ script/release/make-branch | 96 +++++++++++++++++++++++++++++++ script/release/push-release | 71 +++++++++++++++++++++++ script/release/rebase-bump-commit | 39 +++++++++++++ script/release/utils.sh | 20 +++++++ 6 files changed, 269 insertions(+), 42 deletions(-) create mode 100755 script/release/cherry-pick-pr create mode 100755 script/release/make-branch create mode 100755 script/release/push-release create mode 100755 script/release/rebase-bump-commit create mode 100644 script/release/utils.sh diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 966e06ee48..631691a0c6 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -3,11 +3,12 @@ Building a Compose release ## To get started with a new release -1. Create a `bump-$VERSION` branch off master: +Create a branch, update version, and add release notes by running `make-branch` - git checkout -b bump-$VERSION master + git checkout -b bump-$VERSION $BASE_VERSION -2. Merge in the `release` branch on the upstream repo, discarding its tree entirely: +`$BASE_VERSION` will default to master. Use the last version tag for a bug fix +release. git fetch origin git merge --strategy=ours origin/release @@ -58,38 +59,21 @@ Building a Compose release ## To release a version (whether RC or stable) -1. Check that CI is passing on the bump PR. - -2. Check out the bump branch: +Check out the bump branch and run the `push-release` script git checkout bump-$VERSION + ./script/release/push-release $VERSION -3. Build the Linux binary: - script/build-linux +When prompted test the binaries. -4. Build the Mac binary in a Mountain Lion VM: - script/prepare-osx - script/build-osx +1. Draft a release from the tag on GitHub (the script will open the window for + you) -5. Test the binaries and/or get some other people to test them. + In the "Tag version" dropdown, select the tag you just pushed. -6. Create a tag: - - TAG=$VERSION # or $VERSION-rcN, if it's an RC - git tag $TAG - -7. Push the tag to the upstream repo: - - git push git@github.com:docker/compose.git $TAG - -8. Draft a release from the tag on GitHub. - - - Go to https://github.com/docker/compose/releases and click "Draft a new release". - - In the "Tag version" dropdown, select the tag you just pushed. - -9. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: +2. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. @@ -108,24 +92,13 @@ Building a Compose release ...release notes go here... -10. Attach the binaries. +3. Attach the binaries. -11. Don’t publish it just yet! +4. Publish the release on GitHub. -12. Upload the latest version to PyPi: +5. Check that both binaries download (following the install instructions) and run. - python setup.py sdist upload - -13. Check that the pip package installs and runs (best done in a virtualenv): - - pip install -U docker-compose==$TAG - docker-compose version - -14. Publish the release on GitHub. - -15. Check that both binaries download (following the install instructions) and run. - -16. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +6. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) diff --git a/script/release/cherry-pick-pr b/script/release/cherry-pick-pr new file mode 100755 index 0000000000..7062f7aa08 --- /dev/null +++ b/script/release/cherry-pick-pr @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Cherry-pick a PR into the release branch +# + +set -e +set -o pipefail + + +function usage() { + >&2 cat << EOM +Cherry-pick commits from a github pull request. + +Usage: + + $0 +EOM + exit 1 +} + +[ -n "$1" ] || usage + + +REPO=docker/compose +GITHUB=https://github.com/$REPO/pull +PR=$1 +url="$GITHUB/$PR" +hub am -3 $url diff --git a/script/release/make-branch b/script/release/make-branch new file mode 100755 index 0000000000..99f711e164 --- /dev/null +++ b/script/release/make-branch @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Prepare a new release branch +# + +set -e +set -o pipefail + +. script/release/utils.sh + +REPO=git@github.com:docker/compose + + +function usage() { + >&2 cat << EOM +Create a new release branch `release-` + +Usage: + + $0 [] + +Options: + + version version string for the release (ex: 1.6.0) + base_version branch or tag to start from. Defaults to master. For + bug-fix releases use the previous stage release tag. + +EOM + exit 1 +} + +[ -n "$1" ] || usage +VERSION=$1 +BRANCH=bump-$VERSION + +if [ -z "$2" ]; then + BASE_VERSION="master" +else + BASE_VERSION=$2 +fi + + +DEFAULT_REMOTE=release +REMOTE=$(find_remote $REPO) +# If we don't have a docker origin add one +if [ -z "$REMOTE" ]; then + echo "Creating $DEFAULT_REMOTE remote" + git remote add ${DEFAULT_REMOTE} ${REPO} +fi + +# handle the difference between a branch and a tag +if [ -z "$(git name-rev $BASE_VERSION | grep tags)" ]; then + BASE_VERSION=$REMOTE/$BASE_VERSION +fi + +echo "Creating a release branch $VERSION from $BASE_VERSION" +read -n1 -r -p "Continue? (ctrl+c to cancel)" +git fetch $REMOTE -p +git checkout -b $BRANCH $BASE_VERSION + + +# Store the release version for this branch in git, so that other release +# scripts can use it +git config "branch.${BRANCH}.release" $VERSION + + +echo "Update versions in docs/install.md and compose/__init__.py" +# TODO: automate this +$EDITOR docs/install.md +$EDITOR compose/__init__.py + + +echo "Write release notes in CHANGELOG.md" +browser "https://github.com/docker/compose/issues?q=milestone%3A$VERSION+is%3Aclosed" +$EDITOR CHANGELOG.md + + +echo "Verify changes before commit. Exit the shell to commit changes" +git diff +$SHELL +git commit -a -m "Bump $VERSION" --signoff + + +echo "Push branch to user remote" +GITHUB_USER=$USER +USER_REMOTE=$(find_remote $GITHUB_USER/compose) +if [ -z "$USER_REMOTE" ]; then + echo "No user remote found for $GITHUB_USER" + read -n1 -r -p "Enter the name of your github user: " GITHUB_USER + # assumes there is already a user remote somewhere + USER_REMOTE=$(find_remote $GITHUB_USER/compose) +fi + + +git push $USER_REMOTE +browser https://github.com/docker/compose/compare/docker:release...$GITHUB_USER:$BRANCH?expand=1 diff --git a/script/release/push-release b/script/release/push-release new file mode 100755 index 0000000000..276d67c1f4 --- /dev/null +++ b/script/release/push-release @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Create the official release +# + +set -e +set -o pipefail + +. script/release/utils.sh + +function usage() { + >&2 cat << EOM +Publish a release by building all artifacts and pushing them. + +This script requires that 'git config branch.${BRANCH}.release' is set to the +release version for the release branch. + +EOM + exit 1 +} + +BRANCH="$(git rev-parse --abbrev-ref HEAD)" +VERSION="$(git config "branch.${BRANCH}.release")" || usage + +API=https://api.github.com/repos +REPO=docker/compose +GITHUB_REPO=git@github.com:$REPO + +# Check the build status is green +sha=$(git rev-parse HEAD) +url=$API/$REPO/statuses/$sha +build_status=$(curl -s $url | jq -r '.[0].state') +if [[ "$build_status" != "success" ]]; then + >&2 echo "Build status is $build_status, but it should be success." + exit -1 +fi + + +# Build the binaries and sdists +script/build-linux +# TODO: build osx binary +# script/prepare-osx +# script/build-osx +python setup.py sdist --formats=gztar,zip + + +echo "Test those binaries! Exit the shell to continue." +$SHELL + + +echo "Tagging the release as $VERSION" +git tag $VERSION +git push $GITHUB_REPO $VERSION + + +echo "Create a github release" +# TODO: script more of this https://developer.github.com/v3/repos/releases/ +browser https://github.com/$REPO/releases/new + +echo "Uploading sdist to pypi" +python setup.py sdist upload + +echo "Testing pip package" +virtualenv venv-test +source venv-test/bin/activate +pip install docker-compose==$VERSION +docker-compose version +deactivate + +echo "Now publish the github release, and test the downloads." +echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release. diff --git a/script/release/rebase-bump-commit b/script/release/rebase-bump-commit new file mode 100755 index 0000000000..732b319445 --- /dev/null +++ b/script/release/rebase-bump-commit @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Move the "bump to " commit to the HEAD of the branch +# + +set -e + + +function usage() { + >&2 cat << EOM +Move the "bump to " commit to the HEAD of the branch + +This script requires that 'git config branch.${BRANCH}.release' is set to the +release version for the release branch. + +EOM + exit 1 +} + + +BRANCH="$(git rev-parse --abbrev-ref HEAD)" +VERSION="$(git config "branch.${BRANCH}.release")" || usage + + +COMMIT_MSG="Bump $VERSION" +sha=$(git log --grep $COMMIT_MSG --format="%H") +if [ -z "$sha" ]; then + >&2 echo "No commit with message \"$COMMIT_MSG\"" + exit 2 +fi +if [[ "$sha" == "$(git rev-parse HEAD)" ]]; then + >&2 echo "Bump commit already at HEAD" + exit 0 +fi + +commits=$(git log --format="%H" HEAD..$sha | wc -l) + +git rebase --onto $sha~1 HEAD~$commits $BRANCH +git cherry-pick $sha diff --git a/script/release/utils.sh b/script/release/utils.sh new file mode 100644 index 0000000000..d64d116181 --- /dev/null +++ b/script/release/utils.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Util functions for release scritps +# + +set -e + + +function browser() { + local url=$1 + xdg-open $url || open $url +} + + +function find_remote() { + local url=$1 + for remote in $(git remote); do + git config --get remote.${remote}.url | grep $url > /dev/null && echo -n $remote + done +} From dc56e4f97ec83b2b2888b167e4bbb347d4bc9409 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 02:02:01 -0400 Subject: [PATCH 229/337] Update release process docs to use scripts. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 39 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 631691a0c6..c9b7a78cff 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -5,19 +5,18 @@ Building a Compose release Create a branch, update version, and add release notes by running `make-branch` - git checkout -b bump-$VERSION $BASE_VERSION + ./script/release/make-branch $VERSION [$BASE_VERSION] -`$BASE_VERSION` will default to master. Use the last version tag for a bug fix +`$BASE_VERSION` will default to master. Use the last version tag for a bug fix release. - git fetch origin - git merge --strategy=ours origin/release +As part of this script you'll be asked to: -3. Update the version in `docs/install.md` and `compose/__init__.py`. +1. Update the version in `docs/install.md` and `compose/__init__.py`. If the next release will be an RC, append `rcN`, e.g. `1.4.0rc1`. -4. Write release notes in `CHANGES.md`. +2. Write release notes in `CHANGES.md`. Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context where appropriate. @@ -25,38 +24,20 @@ release. Improvements to the code are not worth mentioning. -5. Add a bump commit: - - git commit -am "Bump $VERSION" - -6. Push the bump branch to your fork: - - git push --set-upstream $USERNAME bump-$VERSION - -7. Open a PR from the bump branch against the `release` branch on the upstream repo, **not** against master. ## When a PR is merged into master that we want in the release -1. Check out the bump branch: +1. Check out the bump branch and run the cherry pick script git checkout bump-$VERSION + ./script/release/cherry-pick-pr $PR_NUMBER -2. Cherry-pick the merge commit, fixing any conflicts if necessary: - - git cherry-pick -xm1 $MERGE_COMMIT_HASH - -3. Add a signoff (it’s missing from merge commits): - - git commit --amend --signoff - -4. Move the bump commit back to the tip of the branch: - - git rebase --interactive $PARENT_OF_BUMP_COMMIT - -5. Force-push the bump branch to your fork: +2. When you are done cherry-picking branches move the bump version commit to HEAD + ./script/release/rebase-bump-commit git push --force $USERNAME bump-$VERSION + ## To release a version (whether RC or stable) Check out the bump branch and run the `push-release` script From 1a2a0dd53ded656cc734f454b016c91f0ee2da10 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 10:10:11 -0400 Subject: [PATCH 230/337] Fix some bugs in the release scripts Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 19 +++++++++------ script/release/build-binaries | 21 ++++++++++++++++ script/release/cherry-pick-pr | 6 +++++ script/release/make-branch | 34 +++++++++++++------------- script/release/push-release | 40 +++++++++++++------------------ script/release/rebase-bump-commit | 7 +++--- script/release/utils.sh | 3 +++ 7 files changed, 78 insertions(+), 52 deletions(-) create mode 100755 script/release/build-binaries diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index c9b7a78cff..810c309747 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -7,7 +7,7 @@ Create a branch, update version, and add release notes by running `make-branch` ./script/release/make-branch $VERSION [$BASE_VERSION] -`$BASE_VERSION` will default to master. Use the last version tag for a bug fix +`$BASE_VERSION` will default to master. Use the last version tag for a bug fix release. As part of this script you'll be asked to: @@ -40,15 +40,14 @@ As part of this script you'll be asked to: ## To release a version (whether RC or stable) -Check out the bump branch and run the `push-release` script +Check out the bump branch and run the `build-binary` script git checkout bump-$VERSION - ./script/release/push-release $VERSION + ./script/release/build-binary When prompted test the binaries. - 1. Draft a release from the tag on GitHub (the script will open the window for you) @@ -75,11 +74,17 @@ When prompted test the binaries. 3. Attach the binaries. -4. Publish the release on GitHub. +4. If everything looks good, it's time to push the release. -5. Check that both binaries download (following the install instructions) and run. -6. Email maintainers@dockerproject.org and engineering@docker.com about the new release. + ./script/release/push-release + + +5. Publish the release on GitHub. + +6. Check that both binaries download (following the install instructions) and run. + +7. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) diff --git a/script/release/build-binaries b/script/release/build-binaries new file mode 100755 index 0000000000..9f65b45d27 --- /dev/null +++ b/script/release/build-binaries @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Build the release binaries +# + +. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" + +REPO=docker/compose + +# Build the binaries +script/clean +script/build-linux +# TODO: build osx binary +# script/prepare-osx +# script/build-osx +# TODO: build or fetch the windows binary +echo "You need to build the osx/windows binaries, that step is not automated yet." + +echo "Create a github release" +# TODO: script more of this https://developer.github.com/v3/repos/releases/ +browser https://github.com/$REPO/releases/new diff --git a/script/release/cherry-pick-pr b/script/release/cherry-pick-pr index 7062f7aa08..604600872c 100755 --- a/script/release/cherry-pick-pr +++ b/script/release/cherry-pick-pr @@ -20,6 +20,12 @@ EOM [ -n "$1" ] || usage +if [ -z "$(command -v hub 2> /dev/null)" ]; then + >&2 echo "$0 requires https://hub.github.com/." + >&2 echo "Please install it and ake sure it is available on your \$PATH." + exit 2 +fi + REPO=docker/compose GITHUB=https://github.com/$REPO/pull diff --git a/script/release/make-branch b/script/release/make-branch index 99f711e164..66ed6bbf35 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -3,17 +3,11 @@ # Prepare a new release branch # -set -e -set -o pipefail - -. script/release/utils.sh - -REPO=git@github.com:docker/compose - +. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" function usage() { >&2 cat << EOM -Create a new release branch `release-` +Create a new release branch 'release-' Usage: @@ -29,9 +23,12 @@ EOM exit 1 } + [ -n "$1" ] || usage VERSION=$1 BRANCH=bump-$VERSION +REPO=docker/compose +GITHUB_REPO=git@github.com:$REPO if [ -z "$2" ]; then BASE_VERSION="master" @@ -41,11 +38,11 @@ fi DEFAULT_REMOTE=release -REMOTE=$(find_remote $REPO) +REMOTE="$(find_remote "$GITHUB_REPO")" # If we don't have a docker origin add one if [ -z "$REMOTE" ]; then echo "Creating $DEFAULT_REMOTE remote" - git remote add ${DEFAULT_REMOTE} ${REPO} + git remote add ${DEFAULT_REMOTE} ${GITHUB_REPO} fi # handle the difference between a branch and a tag @@ -65,7 +62,6 @@ git config "branch.${BRANCH}.release" $VERSION echo "Update versions in docs/install.md and compose/__init__.py" -# TODO: automate this $EDITOR docs/install.md $EDITOR compose/__init__.py @@ -75,22 +71,26 @@ browser "https://github.com/docker/compose/issues?q=milestone%3A$VERSION+is%3Acl $EDITOR CHANGELOG.md -echo "Verify changes before commit. Exit the shell to commit changes" git diff -$SHELL -git commit -a -m "Bump $VERSION" --signoff +echo "Verify changes before commit. Exit the shell to commit changes" +$SHELL || true +git commit -a -m "Bump $VERSION" --signoff --no-verify echo "Push branch to user remote" GITHUB_USER=$USER -USER_REMOTE=$(find_remote $GITHUB_USER/compose) +USER_REMOTE="$(find_remote $GITHUB_USER/compose)" if [ -z "$USER_REMOTE" ]; then echo "No user remote found for $GITHUB_USER" - read -n1 -r -p "Enter the name of your github user: " GITHUB_USER + read -r -p "Enter the name of your github user: " GITHUB_USER # assumes there is already a user remote somewhere USER_REMOTE=$(find_remote $GITHUB_USER/compose) fi +if [ -z "$USER_REMOTE" ]; then + >&2 echo "No user remote found. You need to 'git push' your branch." + exit 2 +fi git push $USER_REMOTE -browser https://github.com/docker/compose/compare/docker:release...$GITHUB_USER:$BRANCH?expand=1 +browser https://github.com/$REPO/compare/docker:release...$GITHUB_USER:$BRANCH?expand=1 diff --git a/script/release/push-release b/script/release/push-release index 276d67c1f4..7c44866671 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -3,10 +3,7 @@ # Create the official release # -set -e -set -o pipefail - -. script/release/utils.sh +. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" function usage() { >&2 cat << EOM @@ -22,6 +19,13 @@ EOM BRANCH="$(git rev-parse --abbrev-ref HEAD)" VERSION="$(git config "branch.${BRANCH}.release")" || usage +if [ -z "$(command -v jq 2> /dev/null)" ]; then + >&2 echo "$0 requires https://stedolan.github.io/jq/" + >&2 echo "Please install it and ake sure it is available on your \$PATH." + exit 2 +fi + + API=https://api.github.com/repos REPO=docker/compose GITHUB_REPO=git@github.com:$REPO @@ -35,30 +39,18 @@ if [[ "$build_status" != "success" ]]; then exit -1 fi - -# Build the binaries and sdists -script/build-linux -# TODO: build osx binary -# script/prepare-osx -# script/build-osx -python setup.py sdist --formats=gztar,zip - - -echo "Test those binaries! Exit the shell to continue." -$SHELL - - echo "Tagging the release as $VERSION" git tag $VERSION git push $GITHUB_REPO $VERSION - -echo "Create a github release" -# TODO: script more of this https://developer.github.com/v3/repos/releases/ -browser https://github.com/$REPO/releases/new - echo "Uploading sdist to pypi" -python setup.py sdist upload +python setup.py sdist + +if [ "$(command -v twine 2> /dev/null)" ]; then + twine upload ./dist/docker-compose-${VERSION}.tar.gz +else + python setup.py upload +fi echo "Testing pip package" virtualenv venv-test @@ -68,4 +60,4 @@ docker-compose version deactivate echo "Now publish the github release, and test the downloads." -echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release. +echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release." diff --git a/script/release/rebase-bump-commit b/script/release/rebase-bump-commit index 732b319445..14ad22a982 100755 --- a/script/release/rebase-bump-commit +++ b/script/release/rebase-bump-commit @@ -3,8 +3,7 @@ # Move the "bump to " commit to the HEAD of the branch # -set -e - +. "$(dirname "${BASH_SOURCE[0]}")/utils.sh" function usage() { >&2 cat << EOM @@ -23,7 +22,7 @@ VERSION="$(git config "branch.${BRANCH}.release")" || usage COMMIT_MSG="Bump $VERSION" -sha=$(git log --grep $COMMIT_MSG --format="%H") +sha="$(git log --grep "$COMMIT_MSG" --format="%H")" if [ -z "$sha" ]; then >&2 echo "No commit with message \"$COMMIT_MSG\"" exit 2 @@ -33,7 +32,7 @@ if [[ "$sha" == "$(git rev-parse HEAD)" ]]; then exit 0 fi -commits=$(git log --format="%H" HEAD..$sha | wc -l) +commits=$(git log --format="%H" "$sha..HEAD" | wc -l) git rebase --onto $sha~1 HEAD~$commits $BRANCH git cherry-pick $sha diff --git a/script/release/utils.sh b/script/release/utils.sh index d64d116181..b4e5a2e6a0 100644 --- a/script/release/utils.sh +++ b/script/release/utils.sh @@ -4,6 +4,7 @@ # set -e +set -o pipefail function browser() { @@ -17,4 +18,6 @@ function find_remote() { for remote in $(git remote); do git config --get remote.${remote}.url | grep $url > /dev/null && echo -n $remote done + # Always return true, extra remotes cause it to return false + true } From 04375fd566a7f52485f7c28876dcf433e6c2fa34 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 2 Oct 2015 11:25:16 -0400 Subject: [PATCH 231/337] Restore notes about building non-linux binaries. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 810c309747..30a9805af2 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -45,15 +45,23 @@ Check out the bump branch and run the `build-binary` script git checkout bump-$VERSION ./script/release/build-binary +When prompted build the non-linux binaries and test them. -When prompted test the binaries. +1. Build the Mac binary in a Mountain Lion VM: -1. Draft a release from the tag on GitHub (the script will open the window for + script/prepare-osx + script/build-osx + +2. Download the windows binary from AppVeyor + + https://ci.appveyor.com/project/docker/compose/build//artifacts + +3. Draft a release from the tag on GitHub (the script will open the window for you) In the "Tag version" dropdown, select the tag you just pushed. -2. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: +4. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. @@ -72,19 +80,19 @@ When prompted test the binaries. ...release notes go here... -3. Attach the binaries. +5. Attach the binaries. -4. If everything looks good, it's time to push the release. +6. If everything looks good, it's time to push the release. ./script/release/push-release -5. Publish the release on GitHub. +7. Publish the release on GitHub. -6. Check that both binaries download (following the install instructions) and run. +8. Check that both binaries download (following the install instructions) and run. -7. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +9. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) From 39cea970b8d161ce6986d5ad2f14b63cb3ff3094 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 25 Jul 2015 19:47:36 -0400 Subject: [PATCH 232/337] alpine docker image for running compose and a script to pull and run it with the correct volumes. Signed-off-by: Daniel Nephin --- .dockerignore | 2 +- Dockerfile.run | 15 +++++++++++++++ docs/install.md | 26 ++++++++++++++++++++++---- script/run | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 Dockerfile.run create mode 100755 script/run diff --git a/.dockerignore b/.dockerignore index 5a4da301b1..055ae7ed19 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,6 @@ .tox build coverage-html -dist docs/_site venv +.tox diff --git a/Dockerfile.run b/Dockerfile.run new file mode 100644 index 0000000000..3c12fa1823 --- /dev/null +++ b/Dockerfile.run @@ -0,0 +1,15 @@ + +FROM alpine:edge +RUN apk -U add \ + python \ + py-pip + +COPY requirements.txt /code/requirements.txt +RUN pip install -r /code/requirements.txt + +ENV VERSION 1.4.0dev + +COPY dist/docker-compose-$VERSION.tar.gz /code/docker-compose/ +RUN pip install /code/docker-compose/docker-compose-$VERSION/ + +ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/docs/install.md b/docs/install.md index bc1f8f78c1..fd7b3cabf9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -40,20 +40,38 @@ To install Compose, do the following: curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - If you have problems installing with `curl`, you can use `pip` instead: `pip install -U docker-compose` + If you have problems installing with `curl`, see + [Alternative Install Options](#alternative-install-options). -4. Apply executable permissions to the binary: +5. Apply executable permissions to the binary: $ chmod +x /usr/local/bin/docker-compose -5. Optionally, install [command completion](completion.md) for the +6. Optionally, install [command completion](completion.md) for the `bash` and `zsh` shell. -6. Test the installation. +7. Test the installation. $ docker-compose --version docker-compose version: 1.4.2 + +## Alternative install options + +### Install using pip + + $ sudo pip install -U docker-compose + + +### Install as a container + +Compose can also be run inside a container, from a small bash script wrapper. +To install compose as a container run: + + $ curl -L https://github.com/docker/compose/releases/download/1.5.0/compose-run > /usr/local/bin/docker-compose + $ chmod +x /usr/local/bin/docker-compose + + ## Upgrading If you're upgrading from Compose 1.2 or earlier, you'll need to remove or migrate diff --git a/script/run b/script/run new file mode 100755 index 0000000000..64718efdce --- /dev/null +++ b/script/run @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Run docker-compose in a container +# +# This script will attempt to mirror the host paths by using volumes for the +# following paths: +# * $(pwd) +# * $(dirname $COMPOSE_FILE) if it's set +# * $HOME if it's set +# +# You can add additional volumes (or any docker run options) using +# the $COMPOSE_OPTIONS environment variable. +# + + +set -e + +VERSION="1.4.0dev" +# TODO: move this to an official repo +IMAGE="dnephin/docker-compose:$VERSION" + + +# Setup options for connecting to docker host +if [ -z "$DOCKER_HOST" ]; then + DOCKER_HOST="/var/run/docker.sock" +fi +if [ -S "$DOCKER_HOST" ]; then + DOCKER_ADDR="-v $DOCKER_HOST:$DOCKER_HOST -e DOCKER_HOST" +else + DOCKER_ADDR="-e DOCKER_HOST" +fi + + +# Setup volume mounts for compose config and context +VOLUMES="-v $(pwd):$(pwd)" +if [ -n "$COMPOSE_FILE" ]; then + compose_dir=$(dirname $COMPOSE_FILE) +fi +# TODO: also check --file argument +if [ -n "$compose_dir" ]; then + VOLUMES="$VOLUMES -v $compose_dir:$compose_dir" +fi +if [ -n "$HOME" ]; then + VOLUMES="$VOLUMES -v $HOME:$HOME" +fi + + +exec docker run --rm -ti $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w $(pwd) $IMAGE $@ From e230142a2548ca32da4856bdf35663fad2bf4d27 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 3 Oct 2015 01:24:28 -0400 Subject: [PATCH 233/337] Reduce the scope of sys.stdout patching. Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 51 +++++++++++++++---------------- tests/integration/service_test.py | 12 ++++---- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 5dbe3397f5..3774eb88e6 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -52,32 +52,32 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = old_base_dir # TODO: address the "Inappropriate ioctl for device" warnings in test output - @mock.patch('sys.stdout', new_callable=StringIO) - def test_ps(self, mock_stdout): + def test_ps(self): self.project.get_service('simple').create_container() - self.command.dispatch(['ps'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['ps'], None) self.assertIn('simplecomposefile_simple_1', mock_stdout.getvalue()) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_ps_default_composefile(self, mock_stdout): + def test_ps_default_composefile(self): self.command.base_dir = 'tests/fixtures/multiple-composefiles' - self.command.dispatch(['up', '-d'], None) - self.command.dispatch(['ps'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['up', '-d'], None) + self.command.dispatch(['ps'], None) output = mock_stdout.getvalue() self.assertIn('multiplecomposefiles_simple_1', output) self.assertIn('multiplecomposefiles_another_1', output) self.assertNotIn('multiplecomposefiles_yetanother_1', output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_ps_alternate_composefile(self, mock_stdout): + def test_ps_alternate_composefile(self): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') self._project = get_project(self.command.base_dir, [config_path]) self.command.base_dir = 'tests/fixtures/multiple-composefiles' - self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) - self.command.dispatch(['-f', 'compose2.yml', 'ps'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) + self.command.dispatch(['-f', 'compose2.yml', 'ps'], None) output = mock_stdout.getvalue() self.assertNotIn('multiplecomposefiles_simple_1', output) @@ -105,54 +105,51 @@ class CLITestCase(DockerClientTestCase): mock_logging.info.assert_any_call('Pulling another (nonexisting-image:latest)...') mock_logging.error.assert_any_call('Error: image library/nonexisting-image:latest not found') - @mock.patch('sys.stdout', new_callable=StringIO) - def test_build_plain(self, mock_stdout): + def test_build_plain(self): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) - mock_stdout.truncate(0) cache_indicator = 'Using cache' pull_indicator = 'Status: Image is up to date for busybox:latest' - self.command.dispatch(['build', 'simple'], None) + + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['build', 'simple'], None) output = mock_stdout.getvalue() self.assertIn(cache_indicator, output) self.assertNotIn(pull_indicator, output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_build_no_cache(self, mock_stdout): + def test_build_no_cache(self): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) - mock_stdout.truncate(0) cache_indicator = 'Using cache' pull_indicator = 'Status: Image is up to date for busybox:latest' - self.command.dispatch(['build', '--no-cache', 'simple'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['build', '--no-cache', 'simple'], None) output = mock_stdout.getvalue() self.assertNotIn(cache_indicator, output) self.assertNotIn(pull_indicator, output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_build_pull(self, mock_stdout): + def test_build_pull(self): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) - mock_stdout.truncate(0) cache_indicator = 'Using cache' pull_indicator = 'Status: Image is up to date for busybox:latest' - self.command.dispatch(['build', '--pull', 'simple'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['build', '--pull', 'simple'], None) output = mock_stdout.getvalue() self.assertIn(cache_indicator, output) self.assertIn(pull_indicator, output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_build_no_cache_pull(self, mock_stdout): + def test_build_no_cache_pull(self): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) - mock_stdout.truncate(0) cache_indicator = 'Using cache' pull_indicator = 'Status: Image is up to date for busybox:latest' - self.command.dispatch(['build', '--no-cache', '--pull', 'simple'], None) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + self.command.dispatch(['build', '--no-cache', '--pull', 'simple'], None) output = mock_stdout.getvalue() self.assertNotIn(cache_indicator, output) self.assertIn(pull_indicator, output) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2c9c6fc204..7ea4aae511 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -597,8 +597,7 @@ class ServiceTest(DockerClientTestCase): self.assertNotIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_scale_with_stopped_containers_and_needing_creation(self, mock_stdout): + def test_scale_with_stopped_containers_and_needing_creation(self): """ Given there are some stopped containers and scale is called with a desired number that is greater than the number of stopped containers, @@ -611,7 +610,8 @@ class ServiceTest(DockerClientTestCase): for container in service.containers(): self.assertFalse(container.is_running) - service.scale(2) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + service.scale(2) self.assertEqual(len(service.containers()), 2) for container in service.containers(): @@ -621,8 +621,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_scale_with_api_returns_errors(self, mock_stdout): + def test_scale_with_api_returns_errors(self): """ Test that when scaling if the API returns an error, that error is handled and the remaining threads continue. @@ -635,7 +634,8 @@ class ServiceTest(DockerClientTestCase): 'compose.container.Container.create', side_effect=APIError(message="testing", response={}, explanation="Boom")): - service.scale(3) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + service.scale(3) self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) From aefb7a44b20f51a72f9eead4297403579bed2c4f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 5 Oct 2015 15:48:35 -0400 Subject: [PATCH 234/337] Refactor command class hierarchy to remove an unnecessary intermediate base class Command. Signed-off-by: Daniel Nephin --- compose/cli/command.py | 56 +++++++++++++++-------------------- compose/cli/docopt_command.py | 3 -- compose/cli/main.py | 35 +++++++++++++++------- 3 files changed, 49 insertions(+), 45 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 443b89c611..5c233df722 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import contextlib import logging import os import re @@ -16,7 +17,6 @@ from .. import config from ..project import Project from ..service import ConfigError from .docker_client import docker_client -from .docopt_command import DocoptCommand from .utils import call_silently from .utils import is_mac from .utils import is_ubuntu @@ -24,40 +24,32 @@ from .utils import is_ubuntu log = logging.getLogger(__name__) -class Command(DocoptCommand): - base_dir = '.' - - def dispatch(self, *args, **kwargs): - try: - super(Command, self).dispatch(*args, **kwargs) - except SSLError as e: - raise errors.UserError('SSL error: %s' % e) - except ConnectionError: - if call_silently(['which', 'docker']) != 0: - if is_mac(): - raise errors.DockerNotFoundMac() - elif is_ubuntu(): - raise errors.DockerNotFoundUbuntu() - else: - raise errors.DockerNotFoundGeneric() - elif call_silently(['which', 'boot2docker']) == 0: - raise errors.ConnectionErrorDockerMachine() +@contextlib.contextmanager +def friendly_error_message(): + try: + yield + except SSLError as e: + raise errors.UserError('SSL error: %s' % e) + except ConnectionError: + if call_silently(['which', 'docker']) != 0: + if is_mac(): + raise errors.DockerNotFoundMac() + elif is_ubuntu(): + raise errors.DockerNotFoundUbuntu() else: - raise errors.ConnectionErrorGeneric(self.get_client().base_url) + raise errors.DockerNotFoundGeneric() + elif call_silently(['which', 'boot2docker']) == 0: + raise errors.ConnectionErrorDockerMachine() + else: + raise errors.ConnectionErrorGeneric(self.get_client().base_url) - def perform_command(self, options, handler, command_options): - if options['COMMAND'] in ('help', 'version'): - # Skip looking up the compose file. - handler(None, command_options) - return - project = get_project( - self.base_dir, - get_config_path(options.get('--file')), - project_name=options.get('--project-name'), - verbose=options.get('--verbose')) - - handler(project, command_options) +def project_from_options(base_dir, options): + return get_project( + base_dir, + get_config_path(options.get('--file')), + project_name=options.get('--project-name'), + verbose=options.get('--verbose')) def get_config_path(file_option): diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index 27f4b2bd7f..e3f4aa9e5b 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -25,9 +25,6 @@ class DocoptCommand(object): def dispatch(self, argv, global_options): self.perform_command(*self.parse(argv, global_options)) - def perform_command(self, options, handler, command_options): - handler(command_options) - def parse(self, argv, global_options): options = docopt_full_help(getdoc(self), argv, **self.docopt_options()) command = options['COMMAND'] diff --git a/compose/cli/main.py b/compose/cli/main.py index 60e60b795d..0f0a69cad6 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -23,7 +23,9 @@ from ..project import NoSuchService from ..service import BuildError from ..service import ConvergenceStrategy from ..service import NeedsBuildError -from .command import Command +from .command import friendly_error_message +from .command import project_from_options +from .docopt_command import DocoptCommand from .docopt_command import NoSuchCommand from .errors import UserError from .formatter import Formatter @@ -89,6 +91,15 @@ def setup_logging(): logging.getLogger("requests").propagate = False +def setup_console_handler(verbose): + if verbose: + console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) + console_handler.setLevel(logging.DEBUG) + else: + console_handler.setFormatter(logging.Formatter()) + console_handler.setLevel(logging.INFO) + + # stolen from docopt master def parse_doc_section(name, source): pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)', @@ -96,7 +107,7 @@ def parse_doc_section(name, source): return [s.strip() for s in pattern.findall(source)] -class TopLevelCommand(Command): +class TopLevelCommand(DocoptCommand): """Define and run multi-container applications with Docker. Usage: @@ -130,20 +141,24 @@ class TopLevelCommand(Command): version Show the Docker-Compose version information """ + base_dir = '.' + def docopt_options(self): options = super(TopLevelCommand, self).docopt_options() options['version'] = get_version_info('compose') return options - def perform_command(self, options, *args, **kwargs): - if options.get('--verbose'): - console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) - console_handler.setLevel(logging.DEBUG) - else: - console_handler.setFormatter(logging.Formatter()) - console_handler.setLevel(logging.INFO) + def perform_command(self, options, handler, command_options): + setup_console_handler(options.get('--verbose')) - return super(TopLevelCommand, self).perform_command(options, *args, **kwargs) + if options['COMMAND'] in ('help', 'version'): + # Skip looking up the compose file. + handler(None, command_options) + return + + project = project_from_options(self.base_dir, options) + with friendly_error_message(): + handler(project, command_options) def build(self, project, options): """ From fbaea58fc1a204d676ad098f6b51ba5de1aeccf1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 5 Oct 2015 15:50:16 -0400 Subject: [PATCH 235/337] Fix #2133 - fix call to get_client() Signed-off-by: Daniel Nephin --- compose/cli/command.py | 4 ++-- tests/unit/cli/command_test.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/unit/cli/command_test.py diff --git a/compose/cli/command.py b/compose/cli/command.py index 5c233df722..1a9bc3dcbf 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -38,10 +38,10 @@ def friendly_error_message(): raise errors.DockerNotFoundUbuntu() else: raise errors.DockerNotFoundGeneric() - elif call_silently(['which', 'boot2docker']) == 0: + elif call_silently(['which', 'docker-machine']) == 0: raise errors.ConnectionErrorDockerMachine() else: - raise errors.ConnectionErrorGeneric(self.get_client().base_url) + raise errors.ConnectionErrorGeneric(get_client().base_url) def project_from_options(base_dir, options): diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py new file mode 100644 index 0000000000..0d4324e355 --- /dev/null +++ b/tests/unit/cli/command_test.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import + +import pytest +from requests.exceptions import ConnectionError + +from compose.cli import errors +from compose.cli.command import friendly_error_message +from tests import mock +from tests import unittest + + +class FriendlyErrorMessageTestCase(unittest.TestCase): + + def test_dispatch_generic_connection_error(self): + with pytest.raises(errors.ConnectionErrorGeneric): + with mock.patch( + 'compose.cli.command.call_silently', + autospec=True, + side_effect=[0, 1] + ): + with friendly_error_message(): + raise ConnectionError() From 018b1b1c0f21c8bb76efdc480447c7166e696242 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 6 Oct 2015 12:57:01 +0100 Subject: [PATCH 236/337] Add preparation instructions to Windows build script Signed-off-by: Aanand Prasad --- script/build-windows.ps1 | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 63be086524..f7fd158973 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -1,3 +1,33 @@ +# Builds the Windows binary. +# +# From a fresh 64-bit Windows 10 install, prepare the system as follows: +# +# 1. Install Git: +# +# http://git-scm.com/download/win +# +# 2. Install Python 2.7.10: +# +# https://www.python.org/downloads/ +# +# 3. Append ";C:\Python27;C:\Python27\Scripts" to the "Path" environment variable: +# +# https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx?mfr=true +# +# 4. In Powershell, run the following commands: +# +# $ pip install virtualenv +# $ Set-ExecutionPolicy -Scope CurrentUser RemoteSigned +# +# 5. Clone the repository: +# +# $ git clone https://github.com/docker/compose.git +# $ cd compose +# +# 6. Build the binary: +# +# .\script\build-windows.ps1 + $ErrorActionPreference = "Stop" Set-PSDebug -trace 1 From 5b55a08846088d6cdbc4aca08d3143f5c9c3d3b7 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Sat, 18 Jul 2015 11:38:46 +0200 Subject: [PATCH 237/337] Add support for ro option in volumes_from Fixes #1188 Signed-off-by: Vincent Demeester --- compose/project.py | 24 +++++++++----- compose/service.py | 52 ++++++++++++++++++++++++------- docs/yml.md | 4 ++- tests/integration/project_test.py | 5 +-- tests/integration/service_test.py | 7 +++-- tests/unit/project_test.py | 6 ++-- tests/unit/service_test.py | 34 ++++++++++++++++---- 7 files changed, 97 insertions(+), 35 deletions(-) diff --git a/compose/project.py b/compose/project.py index 4750a7a9ae..919a201f1f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -17,8 +17,10 @@ from .legacy import check_for_legacy_containers from .service import ContainerNet from .service import ConvergenceStrategy from .service import Net +from .service import parse_volume_from_spec from .service import Service from .service import ServiceNet +from .service import VolumeFromSpec from .utils import parallel_execute @@ -34,12 +36,15 @@ def sort_service_dicts(services): def get_service_names(links): return [link.split(':')[0] for link in links] + def get_service_names_from_volumes_from(volumes_from): + return [volume_from.split(':')[0] for volume_from in volumes_from] + def get_service_dependents(service_dict, services): name = service_dict['name'] return [ service for service in services if (name in get_service_names(service.get('links', [])) or - name in service.get('volumes_from', []) or + name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or name == get_service_name_from_net(service.get('net'))) ] @@ -176,20 +181,23 @@ class Project(object): def get_volumes_from(self, service_dict): volumes_from = [] if 'volumes_from' in service_dict: - for volume_name in service_dict.get('volumes_from', []): + for volume_from_config in service_dict.get('volumes_from', []): + volume_from_spec = parse_volume_from_spec(volume_from_config) + # Get service try: - service = self.get_service(volume_name) - volumes_from.append(service) + service_name = self.get_service(volume_from_spec.source) + volume_from_spec = VolumeFromSpec(service_name, volume_from_spec.mode) except NoSuchService: try: - container = Container.from_id(self.client, volume_name) - volumes_from.append(container) + container_name = Container.from_id(self.client, volume_from_spec.source) + volume_from_spec = VolumeFromSpec(container_name, volume_from_spec.mode) except APIError: raise ConfigurationError( 'Service "%s" mounts volumes from "%s", which is ' 'not the name of a service or container.' % ( - service_dict['name'], - volume_name)) + volume_from_config, + volume_from_spec.source)) + volumes_from.append(volume_from_spec) del service_dict['volumes_from'] return volumes_from diff --git a/compose/service.py b/compose/service.py index c9ca00ae41..79a138aac7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -6,7 +6,6 @@ import os import re import sys from collections import namedtuple -from operator import attrgetter import enum import six @@ -82,6 +81,9 @@ class NoSuchImageError(Exception): VolumeSpec = namedtuple('VolumeSpec', 'external internal mode') +VolumeFromSpec = namedtuple('VolumeFromSpec', 'source mode') + + ServiceName = namedtuple('ServiceName', 'project service number') @@ -519,7 +521,7 @@ class Service(object): return [(service.name, alias) for service, alias in self.links] def get_volumes_from_names(self): - return [s.name for s in self.volumes_from if isinstance(s, Service)] + return [s.source.name for s in self.volumes_from if isinstance(s.source, Service)] def get_container_name(self, number, one_off=False): # TODO: Implement issue #652 here @@ -559,16 +561,9 @@ class Service(object): def _get_volumes_from(self): volumes_from = [] - for volume_source in self.volumes_from: - if isinstance(volume_source, Service): - containers = volume_source.containers(stopped=True) - if not containers: - volumes_from.append(volume_source.create_container().id) - else: - volumes_from.extend(map(attrgetter('id'), containers)) - - elif isinstance(volume_source, Container): - volumes_from.append(volume_source.id) + for volume_from_spec in self.volumes_from: + volumes = build_volume_from(volume_from_spec) + volumes_from.extend(volumes) return volumes_from @@ -988,6 +983,39 @@ def parse_volume_spec(volume_config): return VolumeSpec(external, internal, mode) + +def build_volume_from(volume_from_spec): + volumes_from = [] + if isinstance(volume_from_spec.source, Service): + containers = volume_from_spec.source.containers(stopped=True) + if not containers: + volumes_from = ["{}:{}".format(volume_from_spec.source.create_container().id, volume_from_spec.mode)] + else: + volumes_from = ["{}:{}".format(container.id, volume_from_spec.mode) for container in containers] + elif isinstance(volume_from_spec.source, Container): + volumes_from = ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] + return volumes_from + + +def parse_volume_from_spec(volume_from_config): + parts = volume_from_config.split(':') + if len(parts) > 2: + raise ConfigError("Volume %s has incorrect format, should be " + "external:internal[:mode]" % volume_from_config) + + if len(parts) == 1: + source = parts[0] + mode = 'rw' + else: + source, mode = parts + + if mode not in ('rw', 'ro'): + raise ConfigError("VolumeFrom %s has invalid mode (%s), should be " + "one of: rw, ro." % (volume_from_config, mode)) + + return VolumeFromSpec(source, mode) + + # Labels diff --git a/docs/yml.md b/docs/yml.md index 81357df3d6..12c9b554ac 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -346,11 +346,13 @@ should always begin with `.` or `..`. ### volumes_from -Mount all of the volumes from another service or container. +Mount all of the volumes from another service or container, with the +supported flags by docker : ``ro``, ``rw``. volumes_from: - service_name - container_name + - service_name:rw ### cpu\_shares, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, volume\_driver, working\_dir diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index bd7ecccbe8..ff50c80b2a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -6,6 +6,7 @@ from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project from compose.service import ConvergenceStrategy +from compose.service import VolumeFromSpec def build_service_dicts(service_config): @@ -72,7 +73,7 @@ class ProjectTest(DockerClientTestCase): ) db = project.get_service('db') data = project.get_service('data') - self.assertEqual(db.volumes_from, [data]) + self.assertEqual(db.volumes_from, [VolumeFromSpec(data, 'rw')]) def test_volumes_from_container(self): data_container = Container.create( @@ -93,7 +94,7 @@ class ProjectTest(DockerClientTestCase): client=self.client, ) db = project.get_service('db') - self.assertEqual(db.volumes_from, [data_container]) + self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) def test_net_from_service(self): project = Project.from_dicts( diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2c9c6fc204..306060960a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -25,6 +25,7 @@ from compose.service import ConfigError from compose.service import ConvergencePlan from compose.service import Net from compose.service import Service +from compose.service import VolumeFromSpec def create_and_start_container(service, **override_options): @@ -272,12 +273,12 @@ class ServiceTest(DockerClientTestCase): command=["top"], labels={LABEL_PROJECT: 'composetest'}, ) - host_service = self.create_service('host', volumes_from=[volume_service, volume_container_2]) + host_service = self.create_service('host', volumes_from=[VolumeFromSpec(volume_service, 'rw'), VolumeFromSpec(volume_container_2, 'rw')]) host_container = host_service.create_container() host_service.start_container(host_container) - self.assertIn(volume_container_1.id, + self.assertIn(volume_container_1.id + ':rw', host_container.get('HostConfig.VolumesFrom')) - self.assertIn(volume_container_2.id, + self.assertIn(volume_container_2.id + ':rw', host_container.get('HostConfig.VolumesFrom')) def test_execute_convergence_plan_recreate(self): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index ce74eb30b7..f3cf9e2941 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -168,7 +168,7 @@ class ProjectTest(unittest.TestCase): 'volumes_from': ['aaa'] } ], self.mock_client) - self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id]) + self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) def test_use_volumes_from_service_no_container(self): container_name = 'test_vol_1' @@ -191,7 +191,7 @@ class ProjectTest(unittest.TestCase): 'volumes_from': ['vol'] } ], self.mock_client) - self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name]) + self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) @mock.patch.object(Service, 'containers') def test_use_volumes_from_service_container(self, mock_return): @@ -211,7 +211,7 @@ class ProjectTest(unittest.TestCase): 'volumes_from': ['vol'] } ], None) - self.assertEqual(project.get_service('test')._get_volumes_from(), container_ids) + self.assertEqual(project.get_service('test')._get_volumes_from(), [cid + ':rw' for cid in container_ids]) def test_net_unset(self): project = Project.from_dicts('test', [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c682b82377..f85d34d2ac 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -24,6 +24,7 @@ from compose.service import parse_repository_tag from compose.service import parse_volume_spec from compose.service import Service from compose.service import ServiceNet +from compose.service import VolumeFromSpec class ServiceTest(unittest.TestCase): @@ -75,9 +76,18 @@ class ServiceTest(unittest.TestCase): service = Service( 'test', image='foo', - volumes_from=[mock.Mock(id=container_id, spec=Container)]) + volumes_from=[VolumeFromSpec(mock.Mock(id=container_id, spec=Container), 'rw')]) - self.assertEqual(service._get_volumes_from(), [container_id]) + self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) + + def test_get_volumes_from_container_read_only(self): + container_id = 'aabbccddee' + service = Service( + 'test', + image='foo', + volumes_from=[VolumeFromSpec(mock.Mock(id=container_id, spec=Container), 'ro')]) + + self.assertEqual(service._get_volumes_from(), [container_id + ':ro']) def test_get_volumes_from_service_container_exists(self): container_ids = ['aabbccddee', '12345'] @@ -86,9 +96,21 @@ class ServiceTest(unittest.TestCase): mock.Mock(id=container_id, spec=Container) for container_id in container_ids ] - service = Service('test', volumes_from=[from_service], image='foo') + service = Service('test', volumes_from=[VolumeFromSpec(from_service, 'rw')], image='foo') - self.assertEqual(service._get_volumes_from(), container_ids) + self.assertEqual(service._get_volumes_from(), [cid + ":rw" for cid in container_ids]) + + def test_get_volumes_from_service_container_exists_with_flags(self): + for mode in ['ro', 'rw', 'z', 'rw,z', 'z,rw']: + container_ids = ['aabbccddee:' + mode, '12345:' + mode] + from_service = mock.create_autospec(Service) + from_service.containers.return_value = [ + mock.Mock(id=container_id.split(':')[0], spec=Container) + for container_id in container_ids + ] + service = Service('test', volumes_from=[VolumeFromSpec(from_service, mode)], image='foo') + + self.assertEqual(service._get_volumes_from(), container_ids) def test_get_volumes_from_service_no_container(self): container_id = 'abababab' @@ -97,9 +119,9 @@ class ServiceTest(unittest.TestCase): from_service.create_container.return_value = mock.Mock( id=container_id, spec=Container) - service = Service('test', image='foo', volumes_from=[from_service]) + service = Service('test', image='foo', volumes_from=[VolumeFromSpec(from_service, 'rw')]) - self.assertEqual(service._get_volumes_from(), [container_id]) + self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) from_service.create_container.assert_called_once_with() def test_split_domainname_none(self): From fe65c0258d2f3412a18c07e0f701be6b292c2286 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 2 Oct 2015 19:26:45 -0400 Subject: [PATCH 238/337] Remove unused attach_socket function from Container. Signed-off-by: Daniel Nephin --- compose/container.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/compose/container.py b/compose/container.py index 28af093d76..a03acf56fd 100644 --- a/compose/container.py +++ b/compose/container.py @@ -212,9 +212,6 @@ class Container(object): def attach(self, *args, **kwargs): return self.client.attach(self.id, *args, **kwargs) - def attach_socket(self, **kwargs): - return self.client.attach_socket(self.id, **kwargs) - def __repr__(self): return '' % (self.name, self.id[:6]) From 3661e8bc7419ae34e4639edec91df2e1db707312 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 2 Oct 2015 19:47:27 -0400 Subject: [PATCH 239/337] Fix build against the swarm cluster by joining buffered output before parsing json. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 4 ++-- compose/cli/utils.py | 26 ---------------------- compose/progress_stream.py | 6 +----- compose/service.py | 4 +++- compose/utils.py | 38 +++++++++++++++++++++++++++++++++ tests/integration/testcases.py | 6 ++++-- tests/unit/split_buffer_test.py | 2 +- 7 files changed, 49 insertions(+), 37 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 845f799b79..6e1499e1d5 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -6,8 +6,8 @@ from itertools import cycle from . import colors from .multiplexer import Multiplexer -from .utils import split_buffer from compose import utils +from compose.utils import split_buffer class LogPrinter(object): @@ -75,7 +75,7 @@ def build_no_log_generator(container, prefix, color_func): def build_log_generator(container, prefix, color_func): # Attach to container before log printer starts running stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) - line_generator = split_buffer(stream, u'\n') + line_generator = split_buffer(stream) for line in line_generator: yield prefix + line diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 5840f0a8ce..07510e2f31 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -7,7 +7,6 @@ import platform import ssl import subprocess -import six from docker import version as docker_py_version from six.moves import input @@ -36,31 +35,6 @@ def yesno(prompt, default=None): return None -def split_buffer(reader, separator): - """ - Given a generator which yields strings and a separator string, - joins all input, splits on the separator and yields each chunk. - - Unlike string.split(), each chunk includes the trailing - separator, except for the last one if none was found on the end - of the input. - """ - buffered = six.text_type('') - separator = six.text_type(separator) - - for data in reader: - buffered += data.decode('utf-8') - while True: - index = buffered.find(separator) - if index == -1: - break - yield buffered[:index + 1] - buffered = buffered[index + 1:] - - if len(buffered) > 0: - yield buffered - - def call_silently(*args, **kwargs): """ Like subprocess.call(), but redirects stdout and stderr to /dev/null. diff --git a/compose/progress_stream.py b/compose/progress_stream.py index c44b33e561..ca8f351350 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,7 +1,5 @@ import json -import six - from compose import utils @@ -16,9 +14,7 @@ def stream_output(output, stream): lines = {} diff = 0 - for chunk in output: - if six.PY3: - chunk = chunk.decode('utf-8') + for chunk in utils.stream_as_text(output): event = json.loads(chunk) all_events.append(event) diff --git a/compose/service.py b/compose/service.py index c9ca00ae41..bce2e534c9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -33,6 +33,8 @@ from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash from .utils import parallel_execute +from .utils import split_buffer + log = logging.getLogger(__name__) @@ -722,7 +724,7 @@ class Service(object): ) try: - all_events = stream_output(build_output, sys.stdout) + all_events = stream_output(split_buffer(build_output), sys.stdout) except StreamOutputError as e: raise BuildError(self, six.text_type(e)) diff --git a/compose/utils.py b/compose/utils.py index e0304ba506..f201e2d6cf 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -83,6 +83,44 @@ def get_output_stream(stream): return codecs.getwriter('utf-8')(stream) +def stream_as_text(stream): + """Given a stream of bytes or text, if any of the items in the stream + are bytes convert them to text. + + This function can be removed once docker-py returns text streams instead + of byte streams. + """ + for data in stream: + if not isinstance(data, six.text_type): + data = data.decode('utf-8') + yield data + + +def split_buffer(reader, separator=u'\n'): + """ + Given a generator which yields strings and a separator string, + joins all input, splits on the separator and yields each chunk. + + Unlike string.split(), each chunk includes the trailing + separator, except for the last one if none was found on the end + of the input. + """ + buffered = six.text_type('') + separator = six.text_type(separator) + + for data in stream_as_text(reader): + buffered += data + while True: + index = buffered.find(separator) + if index == -1: + break + yield buffered[:index + 1] + buffered = buffered[index + 1:] + + if len(buffered) > 0: + yield buffered + + def write_out_msg(stream, lines, msg_index, msg, status="done"): """ Using special ANSI code characters we can write out the msg over the top of diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 26a0a108a1..7dec3728b8 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -9,6 +9,8 @@ from compose.config.config import ServiceLoader from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service +from compose.utils import split_buffer +from compose.utils import stream_as_text def pull_busybox(client): @@ -71,5 +73,5 @@ class DockerClientTestCase(unittest.TestCase): def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) - build_output = self.client.build(*args, **kwargs) - stream_output(build_output, open('/dev/null', 'w')) + build_output = stream_as_text(self.client.build(*args, **kwargs)) + stream_output(split_buffer(build_output), open('/dev/null', 'w')) diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index 47c72f0865..1775e4cb15 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -2,7 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals from .. import unittest -from compose.cli.utils import split_buffer +from compose.utils import split_buffer class SplitBufferTest(unittest.TestCase): From 15d0c60a73bf700400de826bd122f3f1c30bd0c0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 5 Oct 2015 12:56:10 -0400 Subject: [PATCH 240/337] Fix split buffer with inconsistently delimited json objects. Signed-off-by: Daniel Nephin --- compose/progress_stream.py | 5 +--- compose/service.py | 3 +- compose/utils.py | 52 ++++++++++++++++++++++++++------- tests/integration/testcases.py | 6 ++-- tests/unit/split_buffer_test.py | 2 +- tests/unit/utils_test.py | 16 ++++++++++ 6 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 tests/unit/utils_test.py diff --git a/compose/progress_stream.py b/compose/progress_stream.py index ca8f351350..ac8e4b410f 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,5 +1,3 @@ -import json - from compose import utils @@ -14,8 +12,7 @@ def stream_output(output, stream): lines = {} diff = 0 - for chunk in utils.stream_as_text(output): - event = json.loads(chunk) + for event in utils.json_stream(output): all_events.append(event) if 'progress' in event or 'progressDetail' in event: diff --git a/compose/service.py b/compose/service.py index bce2e534c9..698ab4844f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -33,7 +33,6 @@ from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash from .utils import parallel_execute -from .utils import split_buffer log = logging.getLogger(__name__) @@ -724,7 +723,7 @@ class Service(object): ) try: - all_events = stream_output(split_buffer(build_output), sys.stdout) + all_events = stream_output(build_output, sys.stdout) except StreamOutputError as e: raise BuildError(self, six.text_type(e)) diff --git a/compose/utils.py b/compose/utils.py index f201e2d6cf..c8fddc5f16 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -1,6 +1,7 @@ import codecs import hashlib import json +import json.decoder import logging import sys from threading import Thread @@ -13,6 +14,8 @@ from six.moves.queue import Queue log = logging.getLogger(__name__) +json_decoder = json.JSONDecoder() + def parallel_execute(objects, obj_callable, msg_index, msg): """ @@ -96,29 +99,56 @@ def stream_as_text(stream): yield data -def split_buffer(reader, separator=u'\n'): - """ - Given a generator which yields strings and a separator string, +def line_splitter(buffer, separator=u'\n'): + index = buffer.find(six.text_type(separator)) + if index == -1: + return None, None + return buffer[:index + 1], buffer[index + 1:] + + +def split_buffer(stream, splitter=None, decoder=lambda a: a): + """Given a generator which yields strings and a splitter function, joins all input, splits on the separator and yields each chunk. Unlike string.split(), each chunk includes the trailing separator, except for the last one if none was found on the end of the input. """ + splitter = splitter or line_splitter buffered = six.text_type('') - separator = six.text_type(separator) - for data in stream_as_text(reader): + for data in stream_as_text(stream): buffered += data while True: - index = buffered.find(separator) - if index == -1: + item, rest = splitter(buffered) + if not item: break - yield buffered[:index + 1] - buffered = buffered[index + 1:] - if len(buffered) > 0: - yield buffered + buffered = rest + yield item + + if buffered: + yield decoder(buffered) + + +def json_splitter(buffer): + """Attempt to parse a json object from a buffer. If there is at least one + object, return it and the rest of the buffer, otherwise return None. + """ + try: + obj, index = json_decoder.raw_decode(buffer) + rest = buffer[json.decoder.WHITESPACE.match(buffer, index).end():] + return obj, rest + except ValueError: + return None, None + + +def json_stream(stream): + """Given a stream of text, return a stream of json objects. + This handles streams which are inconsistently buffered (some entries may + be newline delimited, and others are not). + """ + return split_buffer(stream_as_text(stream), json_splitter, json_decoder.decode) def write_out_msg(stream, lines, msg_index, msg, status="done"): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 7dec3728b8..26a0a108a1 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -9,8 +9,6 @@ from compose.config.config import ServiceLoader from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service -from compose.utils import split_buffer -from compose.utils import stream_as_text def pull_busybox(client): @@ -73,5 +71,5 @@ class DockerClientTestCase(unittest.TestCase): def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) - build_output = stream_as_text(self.client.build(*args, **kwargs)) - stream_output(split_buffer(build_output), open('/dev/null', 'w')) + build_output = self.client.build(*args, **kwargs) + stream_output(build_output, open('/dev/null', 'w')) diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index 1775e4cb15..c41ea27d40 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -47,7 +47,7 @@ class SplitBufferTest(unittest.TestCase): self.assert_produces(reader, [string]) def assert_produces(self, reader, expectations): - split = split_buffer(reader(), u'\n') + split = split_buffer(reader()) for (actual, expected) in zip(split, expectations): self.assertEqual(type(actual), type(expected)) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py new file mode 100644 index 0000000000..b272c7349a --- /dev/null +++ b/tests/unit/utils_test.py @@ -0,0 +1,16 @@ +from .. import unittest +from compose import utils + + +class JsonSplitterTestCase(unittest.TestCase): + + def test_json_splitter_no_object(self): + data = '{"foo": "bar' + self.assertEqual(utils.json_splitter(data), (None, None)) + + def test_json_splitter_with_object(self): + data = '{"foo": "bar"}\n \n{"next": "obj"}' + self.assertEqual( + utils.json_splitter(data), + ({'foo': 'bar'}, '{"next": "obj"}') + ) From 6edb6fa262396409839bf1b30c7f7a28651e0125 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 16 Sep 2015 18:58:34 -0400 Subject: [PATCH 241/337] Test against a list of versions generated from docker/docker tags. Signed-off-by: Daniel Nephin --- script/test-versions | 10 ++-- script/versions.py | 139 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 4 deletions(-) create mode 100755 script/versions.py diff --git a/script/test-versions b/script/test-versions index bebc556727..89793359be 100755 --- a/script/test-versions +++ b/script/test-versions @@ -10,13 +10,15 @@ docker run --rm \ --entrypoint="tox" \ "$TAG" -e pre-commit -ALL_DOCKER_VERSIONS="1.7.1 1.8.2" -DEFAULT_DOCKER_VERSION="1.8.2" +get_versions="docker run --rm + --entrypoint=/code/.tox/py27/bin/python + $TAG + /code/script/versions.py docker/docker" if [ "$DOCKER_VERSIONS" == "" ]; then - DOCKER_VERSIONS="$DEFAULT_DOCKER_VERSION" + DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="$ALL_DOCKER_VERSIONS" + DOCKER_VERSIONS="$($get_versions recent -n 2)" fi diff --git a/script/versions.py b/script/versions.py new file mode 100755 index 0000000000..513ca754c0 --- /dev/null +++ b/script/versions.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +""" +Query the github API for the git tags of a project, and return a list of +version tags for recent releases, or the default release. + +The default release is the most recent non-RC version. + +Recent is a list of unqiue major.minor versions, where each is the most +recent version in the series. + +For example, if the list of versions is: + + 1.8.0-rc2 + 1.8.0-rc1 + 1.7.1 + 1.7.0 + 1.7.0-rc1 + 1.6.2 + 1.6.1 + +`default` would return `1.7.1` and +`recent -n 3` would return `1.8.0-rc2 1.7.1 1.6.2` +""" +from __future__ import print_function + +import argparse +import itertools +import operator +from collections import namedtuple + +import requests + + +GITHUB_API = 'https://api.github.com/repos' + + +class Version(namedtuple('_Version', 'major minor patch rc')): + + @classmethod + def parse(cls, version): + version = version.lstrip('v') + version, _, rc = version.partition('-') + major, minor, patch = version.split('.', 3) + return cls(int(major), int(minor), int(patch), rc) + + @property + def major_minor(self): + return self.major, self.minor + + @property + def order(self): + """Return a representation that allows this object to be sorted + correctly with the default comparator. + """ + # rc releases should appear before official releases + rc = (0, self.rc) if self.rc else (1, ) + return (self.major, self.minor, self.patch) + rc + + def __str__(self): + rc = '-{}'.format(self.rc) if self.rc else '' + return '.'.join(map(str, self[:3])) + rc + + +def group_versions(versions): + """Group versions by `major.minor` releases. + + Example: + + >>> group_versions([ + Version(1, 0, 0), + Version(2, 0, 0, 'rc1'), + Version(2, 0, 0), + Version(2, 1, 0), + ]) + + [ + [Version(1, 0, 0)], + [Version(2, 0, 0), Version(2, 0, 0, 'rc1')], + [Version(2, 1, 0)], + ] + """ + return list( + list(releases) + for _, releases + in itertools.groupby(versions, operator.attrgetter('major_minor')) + ) + + +def get_latest_versions(versions, num=1): + """Return a list of the most recent versions for each major.minor version + group. + """ + versions = group_versions(versions) + return [versions[index][0] for index in range(num)] + + +def get_default(versions): + """Return a :class:`Version` for the latest non-rc version.""" + for version in versions: + if not version.rc: + return version + + +def get_github_releases(project): + """Query the Github API for a list of version tags and return them in + sorted order. + + See https://developer.github.com/v3/repos/#list-tags + """ + url = '{}/{}/tags'.format(GITHUB_API, project) + response = requests.get(url) + response.raise_for_status() + versions = [Version.parse(tag['name']) for tag in response.json()] + return sorted(versions, reverse=True, key=operator.attrgetter('order')) + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('project', help="Github project name (ex: docker/docker)") + parser.add_argument('command', choices=['recent', 'default']) + parser.add_argument('-n', '--num', type=int, default=2, + help="Number of versions to return from `recent`") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + versions = get_github_releases(args.project) + + if args.command == 'recent': + print(' '.join(map(str, get_latest_versions(versions, args.num)))) + elif args.command == 'default': + print(get_default(versions)) + else: + raise ValueError("Unknown command {}".format(args.command)) + + +if __name__ == "__main__": + main() From 97dc4895ac76c3517902a290f392e981526aa07c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 6 Oct 2015 11:37:36 -0400 Subject: [PATCH 242/337] Remove unnecessary router.php from wordpress example. Signed-off-by: Daniel Nephin --- docs/wordpress.md | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/docs/wordpress.md b/docs/wordpress.md index 8de5a26441..621459382b 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -55,7 +55,7 @@ and a separate MySQL instance: environment: MYSQL_DATABASE: wordpress -Two supporting files are needed to get this working - first, `wp-config.php` is +A supporting file is needed to get this working. `wp-config.php` is the standard WordPress config file with a single change to point the database configuration at the `db` container: @@ -85,25 +85,6 @@ configuration at the `db` container: require_once(ABSPATH . 'wp-settings.php'); -Second, `router.php` tells PHP's built-in web server how to run WordPress: - - Date: Tue, 6 Oct 2015 15:18:58 -0400 Subject: [PATCH 243/337] Update release scripts for release image. Signed-off-by: Daniel Nephin --- Dockerfile.run | 6 ++---- docs/install.md | 2 +- project/RELEASE-PROCESS.md | 2 +- script/build-image | 16 ++++++++++++++++ script/release/build-binaries | 16 ++++++++++++++++ script/release/push-release | 3 +++ script/{run => run.sh} | 0 7 files changed, 39 insertions(+), 6 deletions(-) create mode 100755 script/build-image rename script/{run => run.sh} (100%) diff --git a/Dockerfile.run b/Dockerfile.run index 3c12fa1823..9f3745fefc 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -7,9 +7,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ENV VERSION 1.4.0dev - -COPY dist/docker-compose-$VERSION.tar.gz /code/docker-compose/ -RUN pip install /code/docker-compose/docker-compose-$VERSION/ +ADD dist/docker-compose-release.tar.gz /code/docker-compose +RUN pip install /code/docker-compose/docker-compose-* ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/docs/install.md b/docs/install.md index fd7b3cabf9..be6a6b26af 100644 --- a/docs/install.md +++ b/docs/install.md @@ -68,7 +68,7 @@ To install Compose, do the following: Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0/compose-run > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 30a9805af2..85bbaf2950 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -80,7 +80,7 @@ When prompted build the non-linux binaries and test them. ...release notes go here... -5. Attach the binaries. +5. Attach the binaries and `script/run.sh` 6. If everything looks good, it's time to push the release. diff --git a/script/build-image b/script/build-image new file mode 100755 index 0000000000..d9faddc7bb --- /dev/null +++ b/script/build-image @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +if [ -z "$1" ]; then + >&2 echo "First argument must be image tag." + exit 1 +fi + +TAG=$1 +VERSION="$(python setup.py --version)" + +python setup.py sdist +cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz +docker build -t docker/compose:$TAG -f Dockerfile.run . + diff --git a/script/release/build-binaries b/script/release/build-binaries index 9f65b45d27..083f8eb589 100755 --- a/script/release/build-binaries +++ b/script/release/build-binaries @@ -5,6 +5,19 @@ . "$(dirname "${BASH_SOURCE[0]}")/utils.sh" +function usage() { + >&2 cat << EOM +Build binaries for the release. + +This script requires that 'git config branch.${BRANCH}.release' is set to the +release version for the release branch. + +EOM + exit 1 +} + +BRANCH="$(git rev-parse --abbrev-ref HEAD)" +VERSION="$(git config "branch.${BRANCH}.release")" || usage REPO=docker/compose # Build the binaries @@ -16,6 +29,9 @@ script/build-linux # TODO: build or fetch the windows binary echo "You need to build the osx/windows binaries, that step is not automated yet." +echo "Building the container distribution" +script/build-image $VERSION + echo "Create a github release" # TODO: script more of this https://developer.github.com/v3/repos/releases/ browser https://github.com/$REPO/releases/new diff --git a/script/release/push-release b/script/release/push-release index 7c44866671..039436da0e 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -46,6 +46,9 @@ git push $GITHUB_REPO $VERSION echo "Uploading sdist to pypi" python setup.py sdist +echo "Uploading the docker image" +docker push docker/compose:$VERSION + if [ "$(command -v twine 2> /dev/null)" ]; then twine upload ./dist/docker-compose-${VERSION}.tar.gz else diff --git a/script/run b/script/run.sh similarity index 100% rename from script/run rename to script/run.sh From 467c73186996465a7bb1e5873ab829d2d1c90f42 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 25 Sep 2015 17:18:46 +0100 Subject: [PATCH 244/337] address PR feedback Signed-off-by: Mazz Mosley --- compose/project.py | 7 +++++-- compose/service.py | 5 +---- tests/integration/service_test.py | 8 +++++++- tests/unit/service_test.py | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/compose/project.py b/compose/project.py index 919a201f1f..999c289041 100644 --- a/compose/project.py +++ b/compose/project.py @@ -37,7 +37,10 @@ def sort_service_dicts(services): return [link.split(':')[0] for link in links] def get_service_names_from_volumes_from(volumes_from): - return [volume_from.split(':')[0] for volume_from in volumes_from] + return [ + parse_volume_from_spec(volume_from).source + for volume_from in volumes_from + ] def get_service_dependents(service_dict, services): name = service_dict['name'] @@ -195,7 +198,7 @@ class Project(object): raise ConfigurationError( 'Service "%s" mounts volumes from "%s", which is ' 'not the name of a service or container.' % ( - volume_from_config, + service_dict['name'], volume_from_spec.source)) volumes_from.append(volume_from_spec) del service_dict['volumes_from'] diff --git a/compose/service.py b/compose/service.py index 79a138aac7..f2f82f6cec 100644 --- a/compose/service.py +++ b/compose/service.py @@ -6,6 +6,7 @@ import os import re import sys from collections import namedtuple +from operator import attrgetter import enum import six @@ -1009,10 +1010,6 @@ def parse_volume_from_spec(volume_from_config): else: source, mode = parts - if mode not in ('rw', 'ro'): - raise ConfigError("VolumeFrom %s has invalid mode (%s), should be " - "one of: rw, ro." % (volume_from_config, mode)) - return VolumeFromSpec(source, mode) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 306060960a..64ce2c6582 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -273,7 +273,13 @@ class ServiceTest(DockerClientTestCase): command=["top"], labels={LABEL_PROJECT: 'composetest'}, ) - host_service = self.create_service('host', volumes_from=[VolumeFromSpec(volume_service, 'rw'), VolumeFromSpec(volume_container_2, 'rw')]) + host_service = self.create_service( + 'host', + volumes_from=[ + VolumeFromSpec(volume_service, 'rw'), + VolumeFromSpec(volume_container_2, 'rw') + ] + ) host_container = host_service.create_container() host_service.start_container(host_container) self.assertIn(volume_container_1.id + ':rw', diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index f85d34d2ac..48e31b11b4 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -379,7 +379,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, net=ServiceNet(Service('other')), links=[(Service('one'), 'one')], - volumes_from=[Service('two')]) + volumes_from=[VolumeFromSpec(Service('two'), 'rw')]) config_dict = service.config_dict() expected = { From 0ff84a78c64b241ffc9cb037db0b17044bb9941d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 6 Oct 2015 13:10:38 +0100 Subject: [PATCH 245/337] Use multiple returns rather than overriding. Also added a doc string for clarity. Signed-off-by: Mazz Mosley --- compose/service.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index f2f82f6cec..a24138540a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -986,16 +986,18 @@ def parse_volume_spec(volume_config): def build_volume_from(volume_from_spec): - volumes_from = [] + """ + volume_from can be either a service or a container. We want to return the + container.id and format it into a string complete with the mode. + """ if isinstance(volume_from_spec.source, Service): containers = volume_from_spec.source.containers(stopped=True) if not containers: - volumes_from = ["{}:{}".format(volume_from_spec.source.create_container().id, volume_from_spec.mode)] + return ["{}:{}".format(volume_from_spec.source.create_container().id, volume_from_spec.mode)] else: - volumes_from = ["{}:{}".format(container.id, volume_from_spec.mode) for container in containers] + return ["{}:{}".format(container.id, volume_from_spec.mode) for container in containers] elif isinstance(volume_from_spec.source, Container): - volumes_from = ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] - return volumes_from + return ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] def parse_volume_from_spec(volume_from_config): From f9028703f4a527dc05302999f8d21c18f84b7055 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 6 Oct 2015 13:11:49 +0100 Subject: [PATCH 246/337] Pick the first container Rather than inefficiently looping through all the containers that a service has and overriding each volumes_from value, pick the first one and return that. Signed-off-by: Mazz Mosley --- compose/service.py | 5 +++-- tests/unit/project_test.py | 2 +- tests/unit/service_test.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index a24138540a..0dbd7f8d1d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -994,8 +994,9 @@ def build_volume_from(volume_from_spec): containers = volume_from_spec.source.containers(stopped=True) if not containers: return ["{}:{}".format(volume_from_spec.source.create_container().id, volume_from_spec.mode)] - else: - return ["{}:{}".format(container.id, volume_from_spec.mode) for container in containers] + + container = containers[0] + return ["{}:{}".format(container.id, volume_from_spec.mode)] elif isinstance(volume_from_spec.source, Container): return ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f3cf9e2941..fc189fbb15 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -211,7 +211,7 @@ class ProjectTest(unittest.TestCase): 'volumes_from': ['vol'] } ], None) - self.assertEqual(project.get_service('test')._get_volumes_from(), [cid + ':rw' for cid in container_ids]) + self.assertEqual(project.get_service('test')._get_volumes_from(), [container_ids[0] + ':rw']) def test_net_unset(self): project = Project.from_dicts('test', [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 48e31b11b4..19d25e2ed5 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -98,7 +98,7 @@ class ServiceTest(unittest.TestCase): ] service = Service('test', volumes_from=[VolumeFromSpec(from_service, 'rw')], image='foo') - self.assertEqual(service._get_volumes_from(), [cid + ":rw" for cid in container_ids]) + self.assertEqual(service._get_volumes_from(), [container_ids[0] + ":rw"]) def test_get_volumes_from_service_container_exists_with_flags(self): for mode in ['ro', 'rw', 'z', 'rw,z', 'z,rw']: @@ -110,7 +110,7 @@ class ServiceTest(unittest.TestCase): ] service = Service('test', volumes_from=[VolumeFromSpec(from_service, mode)], image='foo') - self.assertEqual(service._get_volumes_from(), container_ids) + self.assertEqual(service._get_volumes_from(), [container_ids[0]]) def test_get_volumes_from_service_no_container(self): container_id = 'abababab' From 21a1affc6395a524273d5788cac5e0b7c92a50ce Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 6 Oct 2015 15:43:26 +0100 Subject: [PATCH 247/337] Re-word docs. Signed-off-by: Mazz Mosley --- docs/yml.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 12c9b554ac..a476fd33fa 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -346,8 +346,8 @@ should always begin with `.` or `..`. ### volumes_from -Mount all of the volumes from another service or container, with the -supported flags by docker : ``ro``, ``rw``. +Mount all of the volumes from another service or container, optionally +specifying read-only access(``ro``) or read-write(``rw``). volumes_from: - service_name From 8efc39e616f3cc6f782b83abefe39f778fdf7731 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 7 Oct 2015 14:59:08 +0100 Subject: [PATCH 248/337] Improve boolean warning message. Including examples of more boolean types, eg yes/N as it's not always immediately clear that they are treated as booleans. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 4 ++-- tests/unit/config/config_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 959465e987..0fef304a2a 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -57,9 +57,9 @@ def format_boolean_in_environment(instance): """ if isinstance(instance, bool): log.warn( - "Warning: There is a boolean value, {0} in the 'environment' key.\n" + "Warning: There is a boolean value in the 'environment' key.\n" "Environment variables can only be strings.\nPlease add quotes to any boolean values to make them string " - "(eg, '{0}').\nThis warning will become an error in a future release. \r\n".format(instance) + "(eg, 'True', 'yes', 'N').\nThis warning will become an error in a future release. \r\n" ) return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b505740f57..d3fb4d5f17 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -319,7 +319,7 @@ class ConfigTest(unittest.TestCase): @mock.patch('compose.config.validation.log') def test_logs_warning_for_boolean_in_environment(self, mock_logging): - expected_warning_msg = "Warning: There is a boolean value, True in the 'environment' key." + expected_warning_msg = "Warning: There is a boolean value in the 'environment' key." config.load( build_config_details( {'web': { From 34f5912bbcf5976043840482d13a2d777d40e752 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 7 Oct 2015 11:00:40 -0400 Subject: [PATCH 249/337] Update release script and run.sh image name. Signed-off-by: Daniel Nephin --- script/release/make-branch | 3 ++- script/run.sh | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/script/release/make-branch b/script/release/make-branch index 66ed6bbf35..dde1fb65de 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -61,9 +61,10 @@ git checkout -b $BRANCH $BASE_VERSION git config "branch.${BRANCH}.release" $VERSION -echo "Update versions in docs/install.md and compose/__init__.py" +echo "Update versions in docs/install.md, compose/__init__.py, script/run.sh" $EDITOR docs/install.md $EDITOR compose/__init__.py +$EDITOR script/run.sh echo "Write release notes in CHANGELOG.md" diff --git a/script/run.sh b/script/run.sh index 64718efdce..cf46c143c3 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,9 +15,8 @@ set -e -VERSION="1.4.0dev" -# TODO: move this to an official repo -IMAGE="dnephin/docker-compose:$VERSION" +VERSION="1.5.0" +IMAGE="docker/compose:$VERSION" # Setup options for connecting to docker host From ad96e10938d98cefbbbe1a17774802f36f8b8ad8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 6 Oct 2015 16:32:59 -0400 Subject: [PATCH 250/337] Add travis.yml for building binaries. Signed-off-by: Daniel Nephin --- .travis.yml | 19 +++++++++++++++++++ script/build-osx | 1 - script/prepare-osx | 2 +- script/travis/build-binary | 11 +++++++++++ script/travis/ci | 10 ++++++++++ script/travis/install | 9 +++++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 .travis.yml create mode 100755 script/travis/build-binary create mode 100755 script/travis/ci create mode 100755 script/travis/install diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..0f966f9da6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +sudo: required + +language: python + +services: + - docker + +matrix: + include: + - os: linux + - os: osx + language: generic + + +install: ./script/travis/install + +script: + - ./script/travis/ci + - ./script/travis/build-binary diff --git a/script/build-osx b/script/build-osx index 15a7bbc541..042964e4be 100755 --- a/script/build-osx +++ b/script/build-osx @@ -3,7 +3,6 @@ set -ex PATH="/usr/local/bin:$PATH" -./script/clean rm -rf venv virtualenv -p /usr/local/bin/python venv diff --git a/script/prepare-osx b/script/prepare-osx index ca2776b641..10bbbecc3d 100755 --- a/script/prepare-osx +++ b/script/prepare-osx @@ -24,7 +24,7 @@ if !(which brew); then ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi -brew update +brew update > /dev/null if !(python_version | grep "$desired_python_version"); then if brew list | grep python; then diff --git a/script/travis/build-binary b/script/travis/build-binary new file mode 100755 index 0000000000..b3b7b925be --- /dev/null +++ b/script/travis/build-binary @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + script/build-linux + # TODO: add script/build-image +else + script/prepare-osx + script/build-osx +fi diff --git a/script/travis/ci b/script/travis/ci new file mode 100755 index 0000000000..4cce1bc844 --- /dev/null +++ b/script/travis/ci @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + tox -e py27,py34 -- tests/unit +else + # TODO: we could also install py34 and test against it + python -m tox -e py27 -- tests/unit +fi diff --git a/script/travis/install b/script/travis/install new file mode 100755 index 0000000000..a23667bffc --- /dev/null +++ b/script/travis/install @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + pip install tox==2.1.1 +else + pip install --user tox==2.1.1 +fi From 9ce18849254b29111bfe08bf844e35122b0854e4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 6 Oct 2015 20:50:40 -0400 Subject: [PATCH 251/337] Add upload to bintray from travis. Signed-off-by: Daniel Nephin --- .pre-commit-config.yaml | 1 + .travis.yml | 10 +++++++++ script/build-image | 1 - script/build-linux | 2 +- script/travis/bintray.json.tmpl | 29 ++++++++++++++++++++++++++ script/travis/build-binary | 6 ++++-- script/travis/render-bintray-config.py | 9 ++++++++ 7 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 script/travis/bintray.json.tmpl create mode 100755 script/travis/render-bintray-config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8913a05fd2..3fad8ddcbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,7 @@ - id: check-docstring-first - id: check-merge-conflict - id: check-yaml + - id: check-json - id: debug-statements - id: end-of-file-fixer - id: flake8 diff --git a/.travis.yml b/.travis.yml index 0f966f9da6..3310e2ad9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,13 @@ install: ./script/travis/install script: - ./script/travis/ci - ./script/travis/build-binary + +before_deploy: + - "./script/travis/render-bintray-config.py < ./script/travis/bintray.json.tmpl > ./bintray.json" + +deploy: + provider: bintray + user: docker-compose-roleuser + key: '$BINTRAY_API_KEY' + file: ./bintray.json + skip_cleanup: true diff --git a/script/build-image b/script/build-image index d9faddc7bb..3ac9729b47 100755 --- a/script/build-image +++ b/script/build-image @@ -13,4 +13,3 @@ VERSION="$(python setup.py --version)" python setup.py sdist cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz docker build -t docker/compose:$TAG -f Dockerfile.run . - diff --git a/script/build-linux b/script/build-linux index 4b8696216d..ade18bc535 100755 --- a/script/build-linux +++ b/script/build-linux @@ -5,7 +5,7 @@ set -ex ./script/clean TAG="docker-compose" -docker build -t "$TAG" . +docker build -t "$TAG" . | tail -n 200 docker run \ --rm --entrypoint="script/build-linux-inner" \ -v $(pwd)/dist:/code/dist \ diff --git a/script/travis/bintray.json.tmpl b/script/travis/bintray.json.tmpl new file mode 100644 index 0000000000..7d0adbebcd --- /dev/null +++ b/script/travis/bintray.json.tmpl @@ -0,0 +1,29 @@ +{ + "package": { + "name": "${TRAVIS_OS_NAME}", + "repo": "master", + "subject": "docker-compose", + "desc": "Automated build of master branch from travis ci.", + "website_url": "https://github.com/docker/compose", + "issue_tracker_url": "https://github.com/docker/compose/issues", + "vcs_url": "https://github.com/docker/compose.git", + "licenses": ["Apache-2.0"] + }, + + "version": { + "name": "master", + "desc": "Automated build of the master branch.", + "released": "${DATE}", + "vcs_tag": "master" + }, + + "files": [ + { + "includePattern": "dist/(.*)", + "excludePattern": ".*\.tar.gz", + "uploadPattern": "$1", + "matrixParams": { "override": 1 } + } + ], + "publish": true +} diff --git a/script/travis/build-binary b/script/travis/build-binary index b3b7b925be..0becee7f61 100755 --- a/script/travis/build-binary +++ b/script/travis/build-binary @@ -1,10 +1,12 @@ #!/bin/bash -set -e +set -ex if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then script/build-linux - # TODO: add script/build-image + script/build-image master + # TODO: requires auth + # docker push docker/compose:master else script/prepare-osx script/build-osx diff --git a/script/travis/render-bintray-config.py b/script/travis/render-bintray-config.py new file mode 100755 index 0000000000..6aa468d6dc --- /dev/null +++ b/script/travis/render-bintray-config.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +import datetime +import os.path +import sys + +os.environ['DATE'] = str(datetime.date.today()) + +for line in sys.stdin: + print os.path.expandvars(line), From 23fcace36c38813fbb78b9e06fc805b73d0e8f33 Mon Sep 17 00:00:00 2001 From: ronen barzel Date: Wed, 7 Oct 2015 11:14:53 -0700 Subject: [PATCH 252/337] Bug fix: Use app's Gemfile.lock in Dockerfile The Dockerfile should use the same Gemfile.lock as the app, to make sure the container gets the expected versions of gems installed. Aside from wanting that in principle, without it you can get mysterious gem dependency errors. Here's the scenario: 1. Suppose `Gemfile` includes `gem "some-active-gem", "~> 1.0" 2. When developing the app, you run `bundle install`, which installs the latest version--let's say, 1.0.1-and records it in `Gemfile.lock` 3. Suppose the developers of `some-active-gem` then release v1.0.2 4. Now build the container: docker runs `bundle install`, which installs v1.0.2 and records it in `Gemfile.lock` and then "ADD"s the app worktree, which replaces the `Gemfile.lock` with the one from the worktree that lists v1.0.1. 5. Immediately run your app and it fails with the error `Could not find some-active-gem-1.0.1 in any of the sources` which is a bit befuddling since you just saw it run bundle install so you expect all gem dependencies to be resolved properly. Signed-off-by: ronen barzel --- docs/rails.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/rails.md b/docs/rails.md index 0a164ca75e..105f0f45e0 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -26,6 +26,7 @@ Dockerfile consists of: RUN mkdir /myapp WORKDIR /myapp ADD Gemfile /myapp/Gemfile + ADD Gemfile.lock /myapp/Gemfile.lock RUN bundle install ADD . /myapp From a3eb563f94edfbc7b3341e272f7931b9025496fa Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 8 Oct 2015 11:50:27 +0100 Subject: [PATCH 253/337] Put port ranges back in Signed-off-by: Mazz Mosley --- docs/yml.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/yml.md b/docs/yml.md index a476fd33fa..c3d4a354a0 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -315,9 +315,12 @@ port (a random host port will be chosen). ports: - "3000" + - "3000-3005" - "8000:8000" + - "9090-9091:8080-8081" - "49100:22" - "127.0.0.1:8001:8001" + - "127.0.0.1:5000-5010:5000-5010" ### security_opt From 94e6727831f8a6f1abdb49f5763af2b4cffbae3d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 14:46:31 -0400 Subject: [PATCH 254/337] Re-order docs Makefile for better caching. Signed-off-by: Daniel Nephin --- docs/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index d9add75c15..fcd64900b5 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,11 +1,6 @@ FROM docs/base:latest MAINTAINER Mary Anthony (@moxiegirl) -# To get the git info for this repo -COPY . /src - -COPY . /docs/content/compose/ - RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/docker RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine @@ -13,6 +8,10 @@ RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/tutorials RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content +# To get the git info for this repo +COPY . /src + +COPY . /docs/content/compose/ # Sed to process GitHub Markdown # 1-2 Remove comment code from metadata block From 0e9ec8aa74a57170251b0d0bc6e861218d2bbf67 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 7 Oct 2015 16:47:27 -0400 Subject: [PATCH 255/337] Add publish to bintray step to appveyor.yml Remove Set-PSDebug -trace to prevent the 9000+ lines of debug output from spamming the logs on appveyor. Signed-off-by: Daniel Nephin --- appveyor.yml | 12 ++++++++++-- script/build-windows.ps1 | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index acf8bff34a..b162db1e3a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,12 +9,20 @@ install: # Build the binary after tests build: false +environment: + BINTRAY_USER: "docker-compose-roleuser" + BINTRAY_PATH: "docker-compose/master/windows/master/docker-compose-Windows-x86_64.exe" + test_script: - "tox -e py27,py34 -- tests/unit" - -after_test: - ps: ".\\script\\build-windows.ps1" +deploy_script: + - "curl -sS + -u \"%BINTRAY_USER%:%BINTRAY_API_KEY%\" + -X PUT \"https://api.bintray.com/content/%BINTRAY_PATH%?override=1&publish=1\" + --data-binary @dist\\docker-compose-Windows-x86_64.exe" + artifacts: - path: .\dist\docker-compose-Windows-x86_64.exe name: "Compose Windows binary" diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index f7fd158973..b35fad6f13 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -29,7 +29,6 @@ # .\script\build-windows.ps1 $ErrorActionPreference = "Stop" -Set-PSDebug -trace 1 # Remove virtualenv if (Test-Path venv) { From 6e838b5de17873957ede7068182b620b197d80e7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 10:50:15 -0400 Subject: [PATCH 256/337] Add link to master builds from install docs. Signed-off-by: Daniel Nephin --- docs/install.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/install.md b/docs/install.md index be6a6b26af..4e541b8c3c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -71,6 +71,13 @@ To install compose as a container run: $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose +## Master builds + +If you're interested in trying out a pre-release build you can download a +binary from https://dl.bintray.com/docker-compose/master/. Pre-release +builds allow you to try out new features before they are released, but may +be less stable. + ## Upgrading From cd48a7026a2e8f97ad4d94548e2b59165a398d7a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 17:07:30 -0400 Subject: [PATCH 257/337] Cleanup doc reference links. Removed 'Compose command line completion' and 'Compose environment variables' from the list. command line completion is linked to from install docs, and environment variables are deprecated. Signed-off-by: Daniel Nephin --- docs/completion.md | 1 - docs/django.md | 2 -- docs/env.md | 4 ---- docs/extends.md | 1 - docs/index.md | 13 ------------- docs/install.md | 2 -- docs/production.md | 2 -- docs/rails.md | 2 -- docs/reference/overview.md | 5 ----- docs/wordpress.md | 2 -- docs/yml.md | 2 -- 11 files changed, 36 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index bf8d15551e..891813e911 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -66,4 +66,3 @@ Enjoy working with Compose faster and with less typos! - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) diff --git a/docs/django.md b/docs/django.md index e52f50301d..b11e169358 100644 --- a/docs/django.md +++ b/docs/django.md @@ -131,5 +131,3 @@ example, run `docker-compose up` and in another terminal run: - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/env.md b/docs/env.md index a8e6e214ce..8886548e24 100644 --- a/docs/env.md +++ b/docs/env.md @@ -41,9 +41,5 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - [User guide](/) - [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose command line completion](completion.md) diff --git a/docs/extends.md b/docs/extends.md index 7b4d5b2093..8c35c7a66c 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -360,4 +360,3 @@ locally-defined bindings taking precedence: - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose command line completion](completion.md) diff --git a/docs/index.md b/docs/index.md index 67a6802b06..7900b4f08f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,8 +55,6 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) ## Quick start @@ -218,14 +216,3 @@ like-minded individuals, we have a number of open channels for communication. * To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/project/get-help/). - -## Where to go next - -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) -- [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/install.md b/docs/install.md index be6a6b26af..363a2b290f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -117,5 +117,3 @@ To uninstall Docker Compose if you installed using `pip`: - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/production.md b/docs/production.md index 29e3fd34ec..3e4169e308 100644 --- a/docs/production.md +++ b/docs/production.md @@ -91,5 +91,3 @@ guide. - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/rails.md b/docs/rails.md index 0a164ca75e..c241041005 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -129,5 +129,3 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 9f08246e09..f6496bf786 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -81,9 +81,4 @@ it failed. Defaults to 60 seconds. - [User guide](/) - [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 621459382b..7ac0628999 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -100,5 +100,3 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) diff --git a/docs/yml.md b/docs/yml.md index a476fd33fa..185b31cfc1 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -418,5 +418,3 @@ dollar sign (`$$`). - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](/reference) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) From 182c2537d031f822229eca48b9a2b1985191f573 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 17:22:42 -0400 Subject: [PATCH 258/337] Fix links between reference sections Signed-off-by: Daniel Nephin --- docs/reference/docker-compose.md | 4 ++-- docs/reference/index.md | 4 ++-- docs/reference/overview.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 32fcbe7064..b7cca5b08e 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -100,5 +100,5 @@ directory name. ## Where to go next -* [CLI environment variables](overview.md) -* [Command line reference](index.md) +* [CLI environment variables](/reference/overview.md) +* [Command line reference](/reference) diff --git a/docs/reference/index.md b/docs/reference/index.md index 7a1fb9b444..961dbb8605 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -30,5 +30,5 @@ The following pages describe the usage information for the [docker-compose](/ref ## Where to go next -* [CLI environment variables](overview.md) -* [docker-compose Command](docker-compose.md) +* [CLI environment variables](/reference/overview) +* [docker-compose Command](/reference/docker-compose) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index f6496bf786..019525a581 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -17,8 +17,8 @@ This section describes the subcommands you can use with the `docker-compose` com ## Commands -* [docker-compose Command](docker-compose.md) -* [CLI Reference](index.md) +* [docker-compose Command](/reference/docker-compose.md) +* [CLI Reference](/reference) ## Environment Variables From e90d2b418d7571cf32178d04258f04cecb70dd92 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 18:32:05 -0400 Subject: [PATCH 259/337] Update title for command-line completion docs. Signed-off-by: Daniel Nephin --- docs/completion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 891813e911..30c555c3a7 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -1,6 +1,6 @@ -# Command Completion +# Command-line Completion Compose comes with [command completion](http://en.wikipedia.org/wiki/Command-line_completion) for the bash and zsh shell. From 9b9c8f9cbcfca5458cfe54daff9a953d5969055a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 19:09:01 -0400 Subject: [PATCH 260/337] Clarify irc details, and remove "infancy" statement. Signed-off-by: Daniel Nephin --- docs/index.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7900b4f08f..4b9f29d204 100644 --- a/docs/index.md +++ b/docs/index.md @@ -205,13 +205,14 @@ Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/ ## Getting help -Docker Compose is still in its infancy and under active development. If you need -help, would like to contribute, or simply want to talk about the project with -like-minded individuals, we have a number of open channels for communication. +Docker Compose is under active development. If you need help, would like to +contribute, or simply want to talk about the project with like-minded +individuals, we have a number of open channels for communication. * To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). -* To talk about the project with people in real time: please join the `#docker-compose` channel on IRC. +* To talk about the project with people in real time: please join the + `#docker-compose` channel on freenode IRC. * To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). From bc6b3f970b5fd0fa646dbf166d02d16b556731c5 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 13 Oct 2015 17:03:09 +0100 Subject: [PATCH 261/337] container paths don't need to be expanded They should not ever be relative. Signed-off-by: Mazz Mosley --- compose/config/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9e9cb857fb..373299fd87 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -485,7 +485,6 @@ def resolve_volume_paths(service_dict, working_dir=None): def resolve_volume_path(volume, working_dir, service_name): container_path, host_path = split_path_mapping(volume) - container_path = os.path.expanduser(container_path) if host_path is not None: if host_path.startswith('.'): From 9aaecf95a490436deff190c77546162118145050 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 8 Oct 2015 11:10:51 -0400 Subject: [PATCH 262/337] Update pip install instructions to be more reliable. Signed-off-by: Daniel Nephin --- docs/install.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 7842efb4c3..a701f2985c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -60,7 +60,14 @@ To install Compose, do the following: ### Install using pip - $ sudo pip install -U docker-compose +Compose can be installed from [pypi](https://pypi.python.org/pypi/docker-compose) +using `pip`. If you install using `pip` it is highly recommended that you use a +[virtualenv](https://virtualenv.pypa.io/en/latest/) because many operating systems +have python system packages that conflict with docker-compose dependencies. See +the [virtualenv tutorial](http://docs.python-guide.org/en/latest/dev/virtualenvs/) +to get started. + + $ pip install docker-compose ### Install as a container From c1d5ecaafe3e2e7b1f06342cbeeaef77d72fbac5 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 13 Oct 2015 17:27:25 +0100 Subject: [PATCH 263/337] Workaround splitdrive limitations splitdrive doesn't handle relative paths, so if volume_path contains a relative path, we handle that differently and manually set drive to ''. Signed-off-by: Mazz Mosley --- compose/config/config.py | 13 ++++++++++++- tests/unit/config/config_test.py | 1 - 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 373299fd87..adba3bda50 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -526,7 +526,18 @@ def path_mappings_from_dict(d): def split_path_mapping(volume_path): - drive, volume_config = os.path.splitdrive(volume_path) + """ + Ascertain if the volume_path contains a host path as well as a container + path. Using splitdrive so windows absolute paths won't cause issues with + splitting on ':'. + """ + # splitdrive has limitations when it comes to relative paths, so when it's + # relative, handle special case to set the drive to '' + if volume_path.startswith('.') or volume_path.startswith('~'): + drive, volume_config = '', volume_path + else: + drive, volume_config = os.path.splitdrive(volume_path) + if ':' in volume_config: (host, container) = volume_config.split(':', 1) return (container, drive + host) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d3fb4d5f17..0028210559 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -463,7 +463,6 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/home/me/otherproject:/data']) @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='waiting for this to be resolved: https://github.com/docker/compose/issues/2128') def test_relative_path_does_expand_windows(self): d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) From 0e9c542865215a7ff8333e11e9eaa45b4e5a92c1 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Tue, 13 Oct 2015 04:01:19 -0700 Subject: [PATCH 264/337] Updating to new tooling:supports Github source linking Fixing HEAD Updating to match daniel Fixing the index link Signed-off-by: Mary Anthony --- docs/Dockerfile | 16 ++------- docs/completion.md | 4 +-- docs/django.md | 4 +-- docs/env.md | 6 ++-- docs/extends.md | 2 +- docs/index.md | 4 +-- docs/install.md | 2 +- docs/pre-process.sh | 60 -------------------------------- docs/production.md | 2 +- docs/rails.md | 2 +- docs/reference/docker-compose.md | 4 +-- docs/reference/index.md | 34 +++++++++--------- docs/reference/overview.md | 12 +++---- docs/wordpress.md | 2 +- docs/yml.md | 6 ++-- 15 files changed, 46 insertions(+), 114 deletions(-) delete mode 100755 docs/pre-process.sh diff --git a/docs/Dockerfile b/docs/Dockerfile index fcd64900b5..0114f04e48 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,10 +1,11 @@ -FROM docs/base:latest +FROM docs/base:hugo-github-linking MAINTAINER Mary Anthony (@moxiegirl) -RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/docker +RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/engine RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content/registry +RUN svn checkout https://github.com/kitematic/kitematic/trunk/docs /docs/content/kitematic RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/tutorials RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content @@ -12,14 +13,3 @@ RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content COPY . /src COPY . /docs/content/compose/ - -# Sed to process GitHub Markdown -# 1-2 Remove comment code from metadata block -# 3 Change ](/word to ](/project/ in links -# 4 Change ](word.md) to ](/project/word) -# 5 Remove .md extension from link text -# 6 Change ](../ to ](/project/word) -# 7 Change ](../../ to ](/project/ --> not implemented -# -# -RUN /src/pre-process.sh /docs diff --git a/docs/completion.md b/docs/completion.md index 30c555c3a7..6e7b42c26e 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -59,10 +59,10 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/django.md b/docs/django.md index b11e169358..f4775c4ec4 100644 --- a/docs/django.md +++ b/docs/django.md @@ -124,10 +124,10 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [User guide](/) +- [User guide](../index.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/env.md b/docs/env.md index 8886548e24..984a340bbd 100644 --- a/docs/env.md +++ b/docs/env.md @@ -37,9 +37,9 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` [Docker links]: http://docs.docker.com/userguide/dockerlinks/ -## Compose documentation +## Related Information -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/extends.md b/docs/extends.md index 8c35c7a66c..88fb24a572 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -358,5 +358,5 @@ locally-defined bindings taking precedence: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/index.md b/docs/index.md index 4b9f29d204..bff741b6dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,7 +53,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) ## Quick start @@ -195,7 +195,7 @@ At this point, you have seen the basics of how Compose works. - Next, try the quick start guide for [Django](django.md), [Rails](rails.md), or [WordPress](wordpress.md). -- See the reference guides for complete details on the [commands](/reference), the +- See the reference guides for complete details on the [commands](./reference/index.md), the [configuration file](yml.md) and [environment variables](env.md). ## Release Notes diff --git a/docs/install.md b/docs/install.md index a701f2985c..654f6421d8 100644 --- a/docs/install.md +++ b/docs/install.md @@ -129,5 +129,5 @@ To uninstall Docker Compose if you installed using `pip`: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/pre-process.sh b/docs/pre-process.sh deleted file mode 100755 index f1f6b7fec6..0000000000 --- a/docs/pre-process.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -e - -# Populate an array with just docker dirs and one with content dirs -docker_dir=(`ls -d /docs/content/docker/*`) -content_dir=(`ls -d /docs/content/*`) - -# Loop content not of docker/ -# -# Sed to process GitHub Markdown -# 1-2 Remove comment code from metadata block -# 3 Remove .md extension from link text -# 4 Change ](/ to ](/project/ in links -# 5 Change ](word) to ](/project/word) -# 6 Change ](../../ to ](/project/ -# 7 Change ](../ to ](/project/word) -# -for i in "${content_dir[@]}" -do - : - case $i in - "/docs/content/windows") - ;; - "/docs/content/mac") - ;; - "/docs/content/linux") - ;; - "/docs/content/docker") - y=${i##*/} - find $i -type f -name "*.md" -exec sed -i.old \ - -e '/^/g' \ - -e '/^/g' {} \; - ;; - *) - y=${i##*/} - find $i -type f -name "*.md" -exec sed -i.old \ - -e '/^/g' \ - -e '/^/g' \ - -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/'$y'\//g' \ - -e 's/\(\][(]\)\([A-z].*\)\(\.md\)/\1\/'$y'\/\2/g' \ - -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ - -e 's/\(\][(]\)\(\.\/\)/\1\/'$y'\//g' \ - -e 's/\(\][(]\)\(\.\.\/\.\.\/\)/\1\/'$y'\//g' \ - -e 's/\(\][(]\)\(\.\.\/\)/\1\/'$y'\//g' {} \; - ;; - esac -done - -# -# Move docker directories to content -# -for i in "${docker_dir[@]}" -do - : - if [ -d $i ] - then - mv $i /docs/content/ - fi -done - -rm -rf /docs/content/docker diff --git a/docs/production.md b/docs/production.md index 3e4169e308..5faa1c6962 100644 --- a/docs/production.md +++ b/docs/production.md @@ -89,5 +89,5 @@ guide. - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/rails.md b/docs/rails.md index 3782368d56..74c179b599 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -128,5 +128,5 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index b7cca5b08e..32fcbe7064 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -100,5 +100,5 @@ directory name. ## Where to go next -* [CLI environment variables](/reference/overview.md) -* [Command line reference](/reference) +* [CLI environment variables](overview.md) +* [Command line reference](index.md) diff --git a/docs/reference/index.md b/docs/reference/index.md index 961dbb8605..b2fb5bcadc 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -11,24 +11,24 @@ parent = "smn_compose_ref" ## Compose CLI reference -The following pages describe the usage information for the [docker-compose](/reference/docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. +The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. -* [build](/reference/build.md) -* [help](/reference/help.md) -* [kill](/reference/kill.md) -* [ps](/reference/ps.md) -* [restart](/reference/restart.md) -* [run](/reference/run.md) -* [start](/reference/start.md) -* [up](/reference/up.md) -* [logs](/reference/logs.md) -* [port](/reference/port.md) -* [pull](/reference/pull.md) -* [rm](/reference/rm.md) -* [scale](/reference/scale.md) -* [stop](/reference/stop.md) +* [build](build.md) +* [help](help.md) +* [kill](kill.md) +* [ps](ps.md) +* [restart](restart.md) +* [run](run.md) +* [start](start.md) +* [up](up.md) +* [logs](logs.md) +* [port](port.md) +* [pull](pull.md) +* [rm](rm.md) +* [scale](scale.md) +* [stop](stop.md) ## Where to go next -* [CLI environment variables](/reference/overview) -* [docker-compose Command](/reference/docker-compose) +* [CLI environment variables](overview.md) +* [docker-compose Command](docker-compose.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 019525a581..1a4c268b31 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -17,8 +17,8 @@ This section describes the subcommands you can use with the `docker-compose` com ## Commands -* [docker-compose Command](/reference/docker-compose.md) -* [CLI Reference](/reference) +* [docker-compose Command](docker-compose.md) +* [CLI Reference](index.md) ## Environment Variables @@ -77,8 +77,8 @@ Configures the time (in seconds) a request to the Docker daemon is allowed to ha it failed. Defaults to 60 seconds. -## Compose documentation +## Related Information -- [User guide](/) -- [Installing Compose](install.md) -- [Yaml file reference](yml.md) +- [User guide](../index.md) +- [Installing Compose](../install.md) +- [Yaml file reference](../yml.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 7ac0628999..8c407e4471 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -98,5 +98,5 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) - [Yaml file reference](yml.md) diff --git a/docs/yml.md b/docs/yml.md index 73fa35dfc1..f6ad8b1b75 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -415,9 +415,11 @@ dollar sign (`$$`). ## Compose documentation -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) -- [Command line reference](/reference) +- [Command line reference](./reference/index.md) +- [Compose environment variables](env.md) +- [Compose command line completion](completion.md) From e9ee244e5aab6b686c5b9ae317c37f58d6a6891a Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Tue, 13 Oct 2015 15:06:37 -0700 Subject: [PATCH 265/337] Aaaaaaaaaaaargh Signed-off-by: Mary Anthony --- docs/yml.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index f6ad8b1b75..209d2f180f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -421,5 +421,3 @@ dollar sign (`$$`). - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose environment variables](env.md) -- [Compose command line completion](completion.md) From 5fa5ea0e1637e02e162a4891b4ec84276e573878 Mon Sep 17 00:00:00 2001 From: Charles Chan Date: Sat, 5 Sep 2015 16:44:52 -0700 Subject: [PATCH 266/337] Touchup "Quickstart Guide: Compose and Django" Also incorporated the structural changes by @moxiegirl in PR #1994 as well as subsequent issues reported by @aanand. Signed-off-by: Charles Chan --- docs/django.md | 206 ++++++++++++++++++++++++++++--------------------- 1 file changed, 120 insertions(+), 86 deletions(-) diff --git a/docs/django.md b/docs/django.md index f4775c4ec4..c5e23e762d 100644 --- a/docs/django.md +++ b/docs/django.md @@ -10,124 +10,158 @@ weight=4 -## Quickstart Guide: Compose and Django +# Quickstart Guide: Compose and Django - -This Quick-start Guide will demonstrate how to use Compose to set up and run a +This quick-start guide demonstrates how to use Compose to set up and run a simple Django/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). -### Define the project +## Define the project components -Start by setting up the three files you'll need to build the app. First, since -your app is going to run inside a Docker container containing all of its -dependencies, you'll need to define exactly what needs to be included in the -container. This is done using a file called `Dockerfile`. To begin with, the -Dockerfile consists of: +For this project, you need to create a Dockerfile, a Python dependencies file, +and a `docker-compose.yml` file. - FROM python:2.7 - ENV PYTHONUNBUFFERED 1 - RUN mkdir /code - WORKDIR /code - ADD requirements.txt /code/ - RUN pip install -r requirements.txt - ADD . /code/ +1. Create an empty project directory. -This Dockerfile will define an image that is used to build a container that -includes your application and has Python installed alongside all of your Python -dependencies. For more information on how to write Dockerfiles, see the -[Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + You can name the directory something easy for you to remember. This directory is the context for your application image. The directory should only contain resources to build that image. -Second, you'll define your Python dependencies in a file called -`requirements.txt`: +2. Create a new file called `Dockerfile` in your project directory. - Django - psycopg2 + The Dockerfile defines an application's image content via one or more build + commands that configure that image. Once built, you can run the image in a + container. For more information on `Dockerfiles`, see the [Docker user + guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) + and the [Dockerfile reference](http://docs.docker.com/reference/builder/). -Finally, this is all tied together with a file called `docker-compose.yml`. It -describes the services that comprise your app (here, a web server and database), -which Docker images they use, how they link together, what volumes will be -mounted inside the containers, and what ports they expose. +3. Add the following content to the `Dockerfile`. - db: - image: postgres - web: - build: . - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/code - ports: - - "8000:8000" - links: - - db + FROM python:2.7 + ENV PYTHONUNBUFFERED 1 + RUN mkdir /code + WORKDIR /code + ADD requirements.txt /code/ + RUN pip install -r requirements.txt + ADD . /code/ -See the [`docker-compose.yml` reference](yml.md) for more information on how -this file works. + This `Dockerfile` starts with a Python 2.7 base image. The base image is + modified by adding a new `code` directory. The base image is further modified + by installing the Python requirements defined in the `requirements.txt` file. -### Build the project +4. Save and close the `Dockerfile`. -You can now start a Django project with `docker-compose run`: +5. Create a `requirements.txt` in your project directory. - $ docker-compose run web django-admin.py startproject composeexample . + This file is used by the `RUN pip install -r requirements.txt` command in your `Dockerfile`. -First, Compose will build an image for the `web` service using the `Dockerfile`. -It will then run `django-admin.py startproject composeexample .` inside a -container built using that image. +6. Add the required software in the file. -This will generate a Django app inside the current directory: + Django + psycopg2 - $ ls - Dockerfile docker-compose.yml composeexample manage.py requirements.txt +7. Save and close the `requirements.txt` file. -### Connect the database +8. Create a file called `docker-compose.yml` in your project directory. -Now you need to set up the database connection. Replace the `DATABASES = ...` -definition in `composeexample/settings.py` to read: + The `docker-compose.yml` file describes the services that make your app. In + this example those services are a web server and database. The compose file + also describes which Docker images these services use, how they link + together, any volumes they might need mounted inside the containers. + Finally, the `docker-compose.yml` file describes which ports these services + expose. See the [`docker-compose.yml` reference](yml.md) for more + information on how this file works. - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'postgres', - 'USER': 'postgres', - 'HOST': 'db', - 'PORT': 5432, +9. Add the following configuration to the file. + + db: + image: postgres + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + links: + - db + + This file defines two services: The `db` service and the `web` service. + +10. Save and close the `docker-compose.yml` file. + +## Create a Django project + +In this step, you create a Django started project by building the image from the build context defined in the previous procedure. + +1. Change to the root of your project directory. + +2. Create the Django project using the `docker-compose` command. + + $ docker-compose run web django-admin.py startproject composeexample . + + This instructs Compose to run `django-admin.py startproject composeeexample` + in a container, using the `web` service's image and configuration. Because + the `web` image doesn't exist yet, Compose builds it from the current + directory, as specified by the `build: .` line in `docker-compose.yml`. + + Once the `web` service image is built, Compose runs it and executes the + `django-admin.py startproject` command in the container. This command + instructs Django to create a set of files and directories representing a + Django project. + +3. After the `docker-compose` command completes, list the contents of your project. + + $ ls + Dockerfile docker-compose.yml composeexample manage.py requirements.txt + +## Connect the database + +In this section, you set up the database connection for Django. + +1. In your project dirctory, edit the `composeexample/settings.py` file. + +2. Replace the `DATABASES = ...` with the following: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'postgres', + 'USER': 'postgres', + 'HOST': 'db', + 'PORT': 5432, + } } - } -These settings are determined by the -[postgres](https://registry.hub.docker.com/_/postgres/) Docker image specified -in the Dockerfile. + These settings are determined by the + [postgres](https://registry.hub.docker.com/_/postgres/) Docker image + specified in `docker-compose.yml`. -Then, run `docker-compose up`: +3. Save and close the file. - Recreating myapp_db_1... - Recreating myapp_web_1... - Attaching to myapp_db_1, myapp_web_1 - myapp_db_1 | - myapp_db_1 | PostgreSQL stand-alone backend 9.1.11 - myapp_db_1 | 2014-01-27 12:17:03 UTC LOG: database system is ready to accept connections - myapp_db_1 | 2014-01-27 12:17:03 UTC LOG: autovacuum launcher started - myapp_web_1 | Validating models... - myapp_web_1 | - myapp_web_1 | 0 errors found - myapp_web_1 | January 27, 2014 - 12:12:40 - myapp_web_1 | Django version 1.6.1, using settings 'composeexample.settings' - myapp_web_1 | Starting development server at http://0.0.0.0:8000/ - myapp_web_1 | Quit the server with CONTROL-C. +4. Run the `docker-compose up` command. -Your Django app should nw be running at port 8000 on your Docker daemon. If you are using a Docker Machine VM, you can use the `docker-machine ip MACHINE_NAME` to get the IP address. + $ docker-compose up + Starting composepractice_db_1... + Starting composepractice_web_1... + Attaching to composepractice_db_1, composepractice_web_1 + ... + db_1 | PostgreSQL init process complete; ready for start up. + ... + db_1 | LOG: database system is ready to accept connections + db_1 | LOG: autovacuum launcher started + .. + web_1 | Django version 1.8.4, using settings 'composeexample.settings' + web_1 | Starting development server at http://0.0.0.0:8000/ + web_1 | Quit the server with CONTROL-C. -You can also run management commands with Docker. To set up your database, for -example, run `docker-compose up` and in another terminal run: - - $ docker-compose run web python manage.py syncdb + At this point, your Django app should be running at port `8000` on your + Docker host. If you are using a Docker Machine VM, you can use the + `docker-machine ip MACHINE_NAME` to get the IP address. ## More Compose documentation - [User guide](../index.md) - [Installing Compose](install.md) -- [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [YAML file reference](yml.md) From c7ffbf97c8827025a5d7567cf076a83894eb256a Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Thu, 24 Sep 2015 22:49:38 +0100 Subject: [PATCH 267/337] Extend oneOf error handling. Issue #1989 Signed-off-by: Karol Duleba --- compose/config/validation.py | 18 +++++++++++++++++- tests/unit/config/config_test.py | 12 +++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 959465e987..33b660268c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -4,6 +4,7 @@ import os import sys from functools import wraps +import six from docker.utils.ports import split_port from jsonschema import Draft4Validator from jsonschema import FormatChecker @@ -162,10 +163,25 @@ def process_errors(errors, service_name=None): Inspecting the context value of a ValidationError gives us information about which sub schema failed and which kind of error it is. """ + + required = [context for context in error.context if context.validator == 'required'] + if required: + return required[0].message + + additionalProperties = [context for context in error.context if context.validator == 'additionalProperties'] + if additionalProperties: + invalid_config_key = _parse_key_from_error_msg(additionalProperties[0]) + return "contains unsupported option: '{}'".format(invalid_config_key) + constraint = [context for context in error.context if len(context.path) > 0] if constraint: valid_types = _parse_valid_types_from_validator(constraint[0].validator_value) - msg = "contains {}, which is an invalid type, it should be {}".format( + invalid_config_key = "".join( + "'{}' ".format(fragment) for fragment in constraint[0].path + if isinstance(fragment, six.string_types) + ) + msg = "{}contains {}, which is an invalid type, it should be {}".format( + invalid_config_key, constraint[0].instance, valid_types ) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2c3c5a3a12..7b31038f55 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -335,7 +335,7 @@ class ConfigTest(unittest.TestCase): self.assertTrue(expected_warning_msg in mock_logging.warn.call_args[0][0]) def test_config_invalid_environment_dict_key_raises_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'environment' contains an invalid type" + expected_error_msg = "Service 'web' configuration key 'environment' contains unsupported option: '---'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( @@ -957,7 +957,10 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_invalid_key(self): - expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" + expected_error_msg = ( + "Service 'web' configuration key 'extends' " + "contains unsupported option: 'rogue_key'" + ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( build_config_details( @@ -977,7 +980,10 @@ class ExtendsTest(unittest.TestCase): ) def test_extends_validation_sub_property_key(self): - expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" + expected_error_msg = ( + "Service 'web' configuration key 'extends' 'file' contains 1, " + "which is an invalid type, it should be a string" + ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( build_config_details( From e6344f819a01102248a1c1c6504e38a17eaa9b0e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 11:25:16 -0400 Subject: [PATCH 268/337] Rename yaml reference to compose file reference. Signed-off-by: Daniel Nephin --- docs/completion.md | 2 +- docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 2 +- docs/index.md | 2 +- docs/install.md | 2 +- docs/production.md | 2 +- docs/rails.md | 2 +- docs/reference/overview.md | 2 +- docs/wordpress.md | 2 +- docs/yml.md | 12 ++++++++---- 11 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 6e7b42c26e..0234f0e924 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -65,4 +65,4 @@ Enjoy working with Compose faster and with less typos! - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/django.md b/docs/django.md index c5e23e762d..e6d31ea01c 100644 --- a/docs/django.md +++ b/docs/django.md @@ -164,4 +164,4 @@ In this section, you set up the database connection for Django. - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [YAML file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/env.md b/docs/env.md index 984a340bbd..d32a8ba3f5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -42,4 +42,4 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - [User guide](index.md) - [Installing Compose](install.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/extends.md b/docs/extends.md index 88fb24a572..e9ea207381 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -359,4 +359,4 @@ locally-defined bindings taking precedence: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/index.md b/docs/index.md index bff741b6dd..a881bfa2c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -54,7 +54,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) ## Quick start diff --git a/docs/install.md b/docs/install.md index 654f6421d8..66ccfe7c72 100644 --- a/docs/install.md +++ b/docs/install.md @@ -130,4 +130,4 @@ To uninstall Docker Compose if you installed using `pip`: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/production.md b/docs/production.md index 5faa1c6962..d18beb7b28 100644 --- a/docs/production.md +++ b/docs/production.md @@ -90,4 +90,4 @@ guide. - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/rails.md b/docs/rails.md index 74c179b599..31d5a2253a 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -129,4 +129,4 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 1a4c268b31..51bc39b967 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -81,4 +81,4 @@ it failed. Defaults to 60 seconds. - [User guide](../index.md) - [Installing Compose](../install.md) -- [Yaml file reference](../yml.md) +- [Compose file reference](../yml.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 8c407e4471..b8c8f6b6c4 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -99,4 +99,4 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Yaml file reference](yml.md) +- [Compose file reference](yml.md) diff --git a/docs/yml.md b/docs/yml.md index 209d2f180f..1b97d853e4 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -1,15 +1,19 @@ -# docker-compose.yml reference +# Compose file reference + +The compose file is a [YAML](http://yaml.org/) file where all the top level +keys are the name of a service, and the values are the service definition. +The default path for a compose file is `./docker-compose.yml`. Each service defined in `docker-compose.yml` must specify exactly one of `image` or `build`. Other keys are optional, and are analogous to their From 01f44efe0db34541a77db041fdc2093ca849aba9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 11:32:26 -0400 Subject: [PATCH 269/337] Re-arrange volume_driver in compose file reference. Signed-off-by: Daniel Nephin --- docs/yml.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/yml.md b/docs/yml.md index 1b97d853e4..4f10cf5b2b 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -334,7 +334,7 @@ Override the default labeling scheme for each container. - label:user:USER - label:role:ROLE -### volumes +### volumes, volume\_driver Mount paths as volumes, optionally specifying a path on the host machine (`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). @@ -348,9 +348,19 @@ You can mount a relative path on the host, which will expand relative to the directory of the Compose configuration file being used. Relative paths should always begin with `.` or `..`. +If you use a volume name (instead of a volume path), you may also specify +a `volume_driver`. + + volume_driver: mydriver + + > Note: No path expansion will be done if you have also specified a > `volume_driver`. +See [Docker Volumes](https://docs.docker.com/userguide/dockervolumes/) and +[Volume Plugins](https://docs.docker.com/extend/plugins_volume/) for more +information. + ### volumes_from Mount all of the volumes from another service or container, optionally @@ -361,7 +371,7 @@ specifying read-only access(``ro``) or read-write(``rw``). - container_name - service_name:rw -### cpu\_shares, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, volume\_driver, working\_dir +### cpu\_shares, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. @@ -388,8 +398,6 @@ Each of these is a single value, analogous to its stdin_open: true tty: true - volume_driver: mydriver - ## Variable substitution Your configuration options can contain environment variables. Compose uses the From fbfbe60246de0a86f84da859c07ee1a64e32ea05 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 13 Oct 2015 15:23:27 -0400 Subject: [PATCH 270/337] Rename yml.md to compose-file.md and add an alias for the old url. Signed-off-by: Daniel Nephin --- docs/completion.md | 2 +- docs/{yml.md => compose-file.md} | 1 + docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 2 +- docs/index.md | 4 ++-- docs/install.md | 2 +- docs/production.md | 2 +- docs/rails.md | 2 +- docs/reference/overview.md | 2 +- docs/wordpress.md | 2 +- 11 files changed, 12 insertions(+), 11 deletions(-) rename docs/{yml.md => compose-file.md} (99%) diff --git a/docs/completion.md b/docs/completion.md index 0234f0e924..bc8bedc96c 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -65,4 +65,4 @@ Enjoy working with Compose faster and with less typos! - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/yml.md b/docs/compose-file.md similarity index 99% rename from docs/yml.md rename to docs/compose-file.md index 4f10cf5b2b..725130f789 100644 --- a/docs/yml.md +++ b/docs/compose-file.md @@ -3,6 +3,7 @@ title = "Compose file reference" description = "Compose file reference" keywords = ["fig, composition, compose, docker"] +aliases = ["/compose/yml"] [menu.main] parent="smn_compose_ref" +++ diff --git a/docs/django.md b/docs/django.md index e6d31ea01c..c7ebf58bfe 100644 --- a/docs/django.md +++ b/docs/django.md @@ -164,4 +164,4 @@ In this section, you set up the database connection for Django. - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/env.md b/docs/env.md index d32a8ba3f5..8f3cc3ccb1 100644 --- a/docs/env.md +++ b/docs/env.md @@ -42,4 +42,4 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - [User guide](index.md) - [Installing Compose](install.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/extends.md b/docs/extends.md index e9ea207381..d88ce61c5d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -359,4 +359,4 @@ locally-defined bindings taking precedence: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index a881bfa2c0..2e10e08015 100644 --- a/docs/index.md +++ b/docs/index.md @@ -54,7 +54,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) ## Quick start @@ -196,7 +196,7 @@ At this point, you have seen the basics of how Compose works. - Next, try the quick start guide for [Django](django.md), [Rails](rails.md), or [WordPress](wordpress.md). - See the reference guides for complete details on the [commands](./reference/index.md), the - [configuration file](yml.md) and [environment variables](env.md). + [configuration file](compose-file.md) and [environment variables](env.md). ## Release Notes diff --git a/docs/install.md b/docs/install.md index 66ccfe7c72..2d4d6cadb6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -130,4 +130,4 @@ To uninstall Docker Compose if you installed using `pip`: - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/production.md b/docs/production.md index d18beb7b28..8793f9277e 100644 --- a/docs/production.md +++ b/docs/production.md @@ -90,4 +90,4 @@ guide. - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/rails.md b/docs/rails.md index 31d5a2253a..a33cac26ed 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -129,4 +129,4 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 51bc39b967..3f589a9ded 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -81,4 +81,4 @@ it failed. Defaults to 60 seconds. - [User guide](../index.md) - [Installing Compose](../install.md) -- [Compose file reference](../yml.md) +- [Compose file reference](../compose-file.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index b8c8f6b6c4..8c1f5b0acb 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -99,4 +99,4 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) -- [Compose file reference](yml.md) +- [Compose file reference](compose-file.md) From f4efa293779e7b6a39fef2aab6ebad1b337007bc Mon Sep 17 00:00:00 2001 From: Mohit Soni Date: Wed, 30 Sep 2015 00:25:26 -0700 Subject: [PATCH 271/337] Added support for cgroup_parent This change adds cgroup-parent support to compose project. It allows each service to specify a 'cgroup_parent' option. Signed-off-by: Mohit Soni --- compose/config/config.py | 1 + compose/config/fields_schema.json | 1 + compose/service.py | 5 ++++- docs/compose-file.md | 6 ++++++ tests/unit/service_test.py | 7 +++++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9e9cb857fb..1a995fa8e1 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -21,6 +21,7 @@ from .validation import validate_top_level_object DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', + 'cgroup_parent', 'command', 'cpu_shares', 'cpuset', diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 6fce299cbb..da67774fdf 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -17,6 +17,7 @@ "build": {"type": "string"}, "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, "command": { "oneOf": [ {"type": "string"}, diff --git a/compose/service.py b/compose/service.py index 044b34ad5e..0d89afc02d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -41,6 +41,7 @@ log = logging.getLogger(__name__) DOCKER_START_KEYS = [ 'cap_add', 'cap_drop', + 'cgroup_parent', 'devices', 'dns', 'dns_search', @@ -675,6 +676,7 @@ class Service(object): read_only = options.get('read_only', None) devices = options.get('devices', None) + cgroup_parent = options.get('cgroup_parent', None) return self.client.create_host_config( links=self._get_links(link_to_self=one_off), @@ -696,7 +698,8 @@ class Service(object): read_only=read_only, pid_mode=pid, security_opt=security_opt, - ipc_mode=options.get('ipc') + ipc_mode=options.get('ipc'), + cgroup_parent=cgroup_parent ) def build(self, no_cache=False, pull=False): diff --git a/docs/compose-file.md b/docs/compose-file.md index 725130f789..67322335a7 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -56,6 +56,12 @@ Override the default command. command: bundle exec thin -p 3000 +### cgroup_parent + +Specify an optional parent cgroup for the container. + + cgroup_parent: m-executor-abcd + ### container_name Specify a custom container name, rather than a generated default name. diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 19d25e2ed5..15a9b7c037 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -353,6 +353,13 @@ class ServiceTest(unittest.TestCase): service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) + def test_create_container_no_build_cgroup_parent(self): + service = Service('foo', client=self.mock_client, build='.') + service.image = lambda: {'Id': 'abc123'} + + service.create_container(do_build=False, cgroup_parent='test') + self.assertFalse(self.mock_client.build.called) + def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') service.image = lambda *args, **kwargs: mock_get_image([]) From ca36628a0e3ff1f68a033ca2747cae62fae847c9 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 14 Oct 2015 14:57:37 +0100 Subject: [PATCH 272/337] Test cgroup_parent option is being sent. Signed-off-by: Mazz Mosley --- tests/unit/service_test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 15a9b7c037..84ede75531 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -146,6 +146,18 @@ class ServiceTest(unittest.TestCase): 2000000000 ) + def test_cgroup_parent(self): + self.mock_client.create_host_config.return_value = {} + + service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, cgroup_parent='test') + service._get_container_create_options({'some': 'overrides'}, 1) + + self.assertTrue(self.mock_client.create_host_config.called) + self.assertEqual( + self.mock_client.create_host_config.call_args[1]['cgroup_parent'], + 'test' + ) + def test_log_opt(self): self.mock_client.create_host_config.return_value = {} @@ -353,13 +365,6 @@ class ServiceTest(unittest.TestCase): service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) - def test_create_container_no_build_cgroup_parent(self): - service = Service('foo', client=self.mock_client, build='.') - service.image = lambda: {'Id': 'abc123'} - - service.create_container(do_build=False, cgroup_parent='test') - self.assertFalse(self.mock_client.build.called) - def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') service.image = lambda *args, **kwargs: mock_get_image([]) From 7c6e7e0dced516c860cd5a930217c2bcbcab556b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 13 Oct 2015 14:24:19 -0400 Subject: [PATCH 273/337] Update docker-py to 1.5.0 Signed-off-by: Daniel Nephin --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4f2ea9d14f..daaaa95026 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.4.0 +docker-py==1.5.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/setup.py b/setup.py index 0313fbd052..4020122b15 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.4.0, < 2', + 'docker-py >= 1.5.0, < 2', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From f228173660feb9961e39637d8133cc66c3dc1b33 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 13 Oct 2015 17:31:44 -0400 Subject: [PATCH 274/337] Print docker version. Signed-off-by: Daniel Nephin --- script/ci | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/ci b/script/ci index 58144ea3b9..12dc3c473e 100755 --- a/script/ci +++ b/script/ci @@ -6,7 +6,9 @@ # $ docker build -t "$TAG" . # $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG" -set -e +set -ex + +docker version export DOCKER_VERSIONS=all export DOCKER_DAEMON_ARGS="--storage-driver=overlay" From d5f5eb19243d7c2f04839b232067cdf028ba856d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 7 Oct 2015 16:10:08 +0100 Subject: [PATCH 275/337] Enable use of Docker networking with the --x-networking flag Signed-off-by: Aanand Prasad --- compose/cli/command.py | 13 ++++-- compose/cli/main.py | 4 ++ compose/project.py | 58 +++++++++++++++++++++-- compose/service.py | 5 ++ docs/django.md | 7 +-- docs/index.md | 3 -- docs/networking.md | 84 ++++++++++++++++++++++++++++++++++ docs/rails.md | 4 +- docs/wordpress.md | 2 - script/build-windows.ps1 | 7 ++- tests/integration/cli_test.py | 43 +++++++++++++++++ tests/integration/testcases.py | 11 +++++ tests/unit/service_test.py | 20 ++++++++ tox.ini | 3 +- 14 files changed, 240 insertions(+), 24 deletions(-) create mode 100644 docs/networking.md diff --git a/compose/cli/command.py b/compose/cli/command.py index 1a9bc3dcbf..dd7548b707 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -49,7 +49,10 @@ def project_from_options(base_dir, options): base_dir, get_config_path(options.get('--file')), project_name=options.get('--project-name'), - verbose=options.get('--verbose')) + verbose=options.get('--verbose'), + use_networking=options.get('--x-networking'), + network_driver=options.get('--x-network-driver'), + ) def get_config_path(file_option): @@ -76,14 +79,18 @@ def get_client(verbose=False): return client -def get_project(base_dir, config_path=None, project_name=None, verbose=False): +def get_project(base_dir, config_path=None, project_name=None, verbose=False, + use_networking=False, network_driver=None): config_details = config.find(base_dir, config_path) try: return Project.from_dicts( get_project_name(config_details.working_dir, project_name), config.load(config_details), - get_client(verbose=verbose)) + get_client(verbose=verbose), + use_networking=use_networking, + network_driver=network_driver, + ) except ConfigError as e: raise errors.UserError(six.text_type(e)) diff --git a/compose/cli/main.py b/compose/cli/main.py index 0f0a69cad6..c800d95f98 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -117,6 +117,10 @@ class TopLevelCommand(DocoptCommand): Options: -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) + --x-networking (EXPERIMENTAL) Use new Docker networking functionality. + Requires Docker 1.9 or later. + --x-network-driver DRIVER (EXPERIMENTAL) Specify a network driver (default: "bridge"). + Requires Docker 1.9 or later. --verbose Show more output -v, --version Print version and exit diff --git a/compose/project.py b/compose/project.py index 999c289041..0e20a4cee7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -77,10 +77,12 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client): + def __init__(self, name, services, client, use_networking=False, network_driver=None): self.name = name self.services = services self.client = client + self.use_networking = use_networking + self.network_driver = network_driver or 'bridge' def labels(self, one_off=False): return [ @@ -89,11 +91,15 @@ class Project(object): ] @classmethod - def from_dicts(cls, name, service_dicts, client): + def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None): """ Construct a ServiceCollection from a list of dicts representing services. """ - project = cls(name, [], client) + project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) + + if use_networking: + remove_links(service_dicts) + for service_dict in sort_service_dicts(service_dicts): links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) @@ -103,6 +109,7 @@ class Project(object): Service( client=client, project=name, + use_networking=use_networking, links=links, net=net, volumes_from=volumes_from, @@ -207,6 +214,8 @@ class Project(object): def get_net(self, service_dict): net = service_dict.pop('net', None) if not net: + if self.use_networking: + return Net(self.name) return Net(None) net_name = get_service_name_from_net(net) @@ -289,6 +298,9 @@ class Project(object): plans = self._get_convergence_plans(services, strategy) + if self.use_networking: + self.ensure_network_exists() + return [ container for service in services @@ -350,6 +362,26 @@ class Project(object): return [c for c in containers if matches_service_names(c)] + def get_network(self): + networks = self.client.networks(names=[self.name]) + if networks: + return networks[0] + return None + + def ensure_network_exists(self): + # TODO: recreate network if driver has changed? + if self.get_network() is None: + log.info( + 'Creating network "{}" with driver "{}"' + .format(self.name, self.network_driver) + ) + self.client.create_network(self.name, driver=self.network_driver) + + def remove_network(self): + network = self.get_network() + if network: + self.client.remove_network(network['id']) + def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() @@ -365,6 +397,26 @@ class Project(object): return acc + dep_services +def remove_links(service_dicts): + services_with_links = [s for s in service_dicts if 'links' in s] + if not services_with_links: + return + + if len(services_with_links) == 1: + prefix = '"{}" defines'.format(services_with_links[0]['name']) + else: + prefix = 'Some services ({}) define'.format( + ", ".join('"{}"'.format(s['name']) for s in services_with_links)) + + log.warn( + '\n{} links, which are not compatible with Docker networking and will be ignored.\n' + 'Future versions of Docker will not support links - you should remove them for ' + 'forwards-compatibility.\n'.format(prefix)) + + for s in services_with_links: + del s['links'] + + class NoSuchService(Exception): def __init__(self, name): self.name = name diff --git a/compose/service.py b/compose/service.py index 0d89afc02d..5f1d59468c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -113,6 +113,7 @@ class Service(object): name, client=None, project='default', + use_networking=False, links=None, volumes_from=None, net=None, @@ -124,6 +125,7 @@ class Service(object): self.name = name self.client = client self.project = project + self.use_networking = use_networking self.links = links or [] self.volumes_from = volumes_from or [] self.net = net or Net(None) @@ -602,6 +604,9 @@ class Service(object): container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] + if 'hostname' not in container_options and self.use_networking: + container_options['hostname'] = self.name + if 'ports' in container_options or 'expose' in self.options: ports = [] all_ports = container_options.get('ports', []) + self.options.get('expose', []) diff --git a/docs/django.md b/docs/django.md index c7ebf58bfe..2ebf4b4b97 100644 --- a/docs/django.md +++ b/docs/django.md @@ -64,9 +64,8 @@ and a `docker-compose.yml` file. The `docker-compose.yml` file describes the services that make your app. In this example those services are a web server and database. The compose file - also describes which Docker images these services use, how they link - together, any volumes they might need mounted inside the containers. - Finally, the `docker-compose.yml` file describes which ports these services + also describes which Docker images these services use, any volumes they might + need mounted inside the containers, and any ports they might expose. See the [`docker-compose.yml` reference](yml.md) for more information on how this file works. @@ -81,8 +80,6 @@ and a `docker-compose.yml` file. - .:/code ports: - "8000:8000" - links: - - db This file defines two services: The `db` service and the `web` service. diff --git a/docs/index.md b/docs/index.md index 2e10e08015..e19e7d7f44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -128,8 +128,6 @@ Next, define a set of services using `docker-compose.yml`: - "5000:5000" volumes: - .:/code - links: - - redis redis: image: redis @@ -138,7 +136,6 @@ This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. * Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. -* Links the web container to the Redis service. The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. diff --git a/docs/networking.md b/docs/networking.md new file mode 100644 index 0000000000..f4227917ac --- /dev/null +++ b/docs/networking.md @@ -0,0 +1,84 @@ + + + +# Networking in Compose + +> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. + +Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. + +> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it. + +For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: + + web: + build: . + ports: + - "8000:8000" + db: + image: postgres + +When you run `docker-compose --x-networking up`, the following happens: + +1. A network called `myapp` is created. +2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`. +3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`. + +Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database. + +Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. + +## Updating containers + +If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. + +If any containers have connections open to the old container, they will be closed. It is a container's responsibility to detect this condition, look up the name again and reconnect. + +## Configure how services are published + +By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option: + + web: + build: . + hostname: "my-web-application" + +This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`. + +## Scaling services + +If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one. + +This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner. + +## Links + +Docker links are a one-way, single-host communication system. They should now be considered deprecated, and you should update your app to use networking instead. In the majority of cases, this will simply involve removing the `links` sections from your `docker-compose.yml`. + +## Specifying the network driver + +By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs). + +You can specify which one to use with the `--x-network-driver` flag: + + $ docker-compose --x-networking --x-network-driver=overlay up + +## Multi-host networking + +(TODO: talk about Swarm and the overlay driver) + +## Custom container network modes + +Compose allows you to specify a custom network mode for a service with the `net` option - for example, `net: "host"` specifies that its containers should use the same network namespace as the Docker host, and `net: "none"` specifies that they should have no networking capabilities. + +If a service specifies the `net` option, its containers will *not* join the app’s network and will not be able to communicate with other services in the app. + +If *all* services in an app specify the `net` option, a network will not be created at all. diff --git a/docs/rails.md b/docs/rails.md index a33cac26ed..9801ef7419 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,7 +37,7 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. +Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port. db: image: postgres @@ -48,8 +48,6 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th - .:/myapp ports: - "3000:3000" - links: - - db ### Build the project diff --git a/docs/wordpress.md b/docs/wordpress.md index 8c1f5b0acb..5c9bcdbd9f 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -46,8 +46,6 @@ and a separate MySQL instance: command: php -S 0.0.0.0:8000 -t /code ports: - "8000:8000" - links: - - db volumes: - .:/code db: diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index b35fad6f13..6e8a7c5ae7 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -42,15 +42,14 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -.\venv\Scripts\pip install pypiwin32==219 -.\venv\Scripts\pip install -r requirements.txt -.\venv\Scripts\pip install --no-deps . - # TODO: pip warns when installing from a git sha, so we need to set ErrorAction to # 'Continue'. See # https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 # This can be removed once pyinstaller 3.x is released and we upgrade $ErrorActionPreference = "Continue" +.\venv\Scripts\pip install pypiwin32==219 +.\venv\Scripts\pip install -r requirements.txt +.\venv\Scripts\pip install --no-deps . .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt # Build binary diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 3774eb88e6..a18b69f500 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -185,6 +185,49 @@ class CLITestCase(DockerClientTestCase): set(self.project.containers()) ) + def test_up_without_networking(self): + self.require_engine_version("1.9") + + self.command.base_dir = 'tests/fixtures/links-composefile' + self.command.dispatch(['up', '-d'], None) + + networks = [n for n in self.client.networks(names=[self.project.name])] + self.assertEqual(len(networks), 0) + + for service in self.project.get_services(): + containers = service.containers() + self.assertEqual(len(containers), 1) + self.assertNotEqual(containers[0].get('Config.Hostname'), service.name) + + web_container = self.project.get_service('web').containers()[0] + self.assertTrue(web_container.get('HostConfig.Links')) + + def test_up_with_networking(self): + self.require_engine_version("1.9") + + self.command.base_dir = 'tests/fixtures/links-composefile' + self.command.dispatch(['--x-networking', 'up', '-d'], None) + + services = self.project.get_services() + + networks = [n for n in self.client.networks(names=[self.project.name])] + for n in networks: + self.addCleanup(self.client.remove_network, n['id']) + self.assertEqual(len(networks), 1) + self.assertEqual(networks[0]['driver'], 'bridge') + + network = self.client.inspect_network(networks[0]['id']) + self.assertEqual(len(network['containers']), len(services)) + + for service in services: + containers = service.containers() + self.assertEqual(len(containers), 1) + self.assertIn(containers[0].id, network['containers']) + self.assertEqual(containers[0].get('Config.Hostname'), service.name) + + web_container = self.project.get_service('web').containers()[0] + self.assertFalse(web_container.get('HostConfig.Links')) + def test_up_with_links(self): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d', 'web'], None) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 26a0a108a1..a412fb04fb 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from __future__ import unicode_literals from docker import errors +from docker.utils import version_lt +from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client @@ -73,3 +75,12 @@ class DockerClientTestCase(unittest.TestCase): kwargs.setdefault('rm', True) build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) + + def require_engine_version(self, minimum): + # Drop '-dev' or '-rcN' suffix + engine = self.client.version()['Version'].split('-', 1)[0] + if version_lt(engine, minimum): + skip( + "Engine version is too low ({} < {})" + .format(engine, minimum) + ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 84ede75531..c5e1a9fb06 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -203,6 +203,26 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['hostname'], 'name.sub', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') + def test_no_default_hostname_when_not_using_networking(self): + service = Service( + 'foo', + image='foo', + use_networking=False, + client=self.mock_client, + ) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertIsNone(opts.get('hostname')) + + def test_hostname_defaults_to_service_name_when_using_networking(self): + service = Service( + 'foo', + image='foo', + use_networking=True, + client=self.mock_client, + ) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['hostname'], 'foo') + def test_get_container_create_options_with_name_option(self): service = Service( 'foo', diff --git a/tox.ini b/tox.ini index dbf639201b..f05c5ed260 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,10 @@ passenv = setenv = HOME=/tmp deps = + -rrequirements.txt -rrequirements-dev.txt commands = - py.test -v \ + py.test -v -rxs \ --cov=compose \ --cov-report html \ --cov-report term \ From e2f792c4f43314fed2dca0f5a06ce7fecebead64 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 12:09:50 -0400 Subject: [PATCH 276/337] If -x-networking is used, set the correct API version. Signed-off-by: Daniel Nephin --- compose/cli/command.py | 7 ++++--- compose/cli/docker_client.py | 9 +++++++-- tests/integration/cli_test.py | 11 +++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index dd7548b707..525217ee75 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -67,8 +67,8 @@ def get_config_path(file_option): return [config_file] if config_file else None -def get_client(verbose=False): - client = docker_client() +def get_client(verbose=False, version=None): + client = docker_client(version=version) if verbose: version_info = six.iteritems(client.version()) log.info("Compose version %s", __version__) @@ -83,11 +83,12 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False, use_networking=False, network_driver=None): config_details = config.find(base_dir, config_path) + api_version = '1.21' if use_networking else None try: return Project.from_dicts( get_project_name(config_details.working_dir, project_name), config.load(config_details), - get_client(verbose=verbose), + get_client(verbose=verbose, version=api_version), use_networking=use_networking, network_driver=network_driver, ) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 2c634f3376..734f4237b0 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -9,7 +9,10 @@ from ..const import HTTP_TIMEOUT log = logging.getLogger(__name__) -def docker_client(): +DEFAULT_API_VERSION = '1.19' + + +def docker_client(version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. @@ -18,6 +21,8 @@ def docker_client(): log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') kwargs = kwargs_from_env(assert_hostname=False) - kwargs['version'] = os.environ.get('COMPOSE_API_VERSION', '1.19') + kwargs['version'] = version or os.environ.get( + 'COMPOSE_API_VERSION', + DEFAULT_API_VERSION) kwargs['timeout'] = HTTP_TIMEOUT return Client(**kwargs) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index a18b69f500..78519d1418 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -10,6 +10,7 @@ from six import StringIO from .. import mock from .testcases import DockerClientTestCase from compose.cli.command import get_project +from compose.cli.docker_client import docker_client from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.project import NoSuchService @@ -190,8 +191,9 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d'], None) + client = docker_client(version='1.21') - networks = [n for n in self.client.networks(names=[self.project.name])] + networks = client.networks(names=[self.project.name]) self.assertEqual(len(networks), 0) for service in self.project.get_services(): @@ -207,16 +209,17 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['--x-networking', 'up', '-d'], None) + client = docker_client(version='1.21') services = self.project.get_services() - networks = [n for n in self.client.networks(names=[self.project.name])] + networks = client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(self.client.remove_network, n['id']) + self.addCleanup(client.remove_network, n['id']) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['driver'], 'bridge') - network = self.client.inspect_network(networks[0]['id']) + network = client.inspect_network(networks[0]['id']) self.assertEqual(len(network['containers']), len(services)) for service in services: From 709bd9c363c1f48138ab671b9505569057b5e04b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 12:37:43 -0400 Subject: [PATCH 277/337] Bump 1.5.0rc1 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 48 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 4 ++-- script/run.sh | 2 +- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 598f5e5794..730cd30ef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,54 @@ Change log ========== +1.5.0 (2015-10-13) +------------------ + +Major Features + +- Compose is now available on windows. +- Environment variable can be used in the compose file. See + https://github.com/docker/compose/blob/129092b7/docs/yml.md#variable-substitution +- Multiple compose files can be specified, allowing you to override + setting in the default compose file. See + https://github.com/docker/compose/blob/129092b7/docs/reference/docker-compose.md + for more details. +- Configuration validation is now a lot more strict +- `up` now waits for all services to exit before shutting down +- Support for the new docker networking can be enabled with + the `--x-networking` flag + +New Features + +- `volumes_from` now supports a mode option allowing for read-only + `volumes_from` +- Volumes that don't start with a path indicator (`.` or `/`) will now be + treated as a named volume. Previously this was a warning. +- `--pull` flag added to `build` +- `--ignore-pull-failures` flag added to `pull` +- Support for the `ipc` field added to the compose file +- Containers created by `run` can now be named with the `--name` flag +- If you install Compose with pip or use it as a library, it now + works with Python 3 +- `image` field now supports image digests (in addition to ids and tags) +- `ports` now supports ranges of ports +- `--publish` flag added to `run` +- New subcommands `pause` and `unpause` +- services may be extended from the same file without a `file` key in + `extends` +- Compose can be installed and run as a docker image. This is an experimental + feature. + + +Bug Fixes + +- Support all `log_drivers` +- Fixed `build` when running against swarm +- `~` is no longer expanded on the host when included as part of a container + volume path + + + 1.4.2 (2015-09-22) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index e3ace98356..06897c8448 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0dev' +__version__ = '1.5.0rc1' diff --git a/docs/install.md b/docs/install.md index 2d4d6cadb6..31b2ccad47 100644 --- a/docs/install.md +++ b/docs/install.md @@ -53,7 +53,7 @@ To install Compose, do the following: 7. Test the installation. $ docker-compose --version - docker-compose version: 1.4.2 + docker-compose version: 1.5.0rc1 ## Alternative install options @@ -75,7 +75,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc1/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index cf46c143c3..68ee4faa5b 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0" +VERSION="1.5.0rc1" IMAGE="docker/compose:$VERSION" From 49ca23c0347a99670a156b8cf1d1f9647d664814 Mon Sep 17 00:00:00 2001 From: Tim Butler Date: Thu, 15 Oct 2015 17:09:57 +1000 Subject: [PATCH 278/337] Fix link to Release Process doc in README.md Signed-off-by: Tim Butler --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c776a71c2..d779d607c3 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ Want to help build Compose? Check out our [contributing documentation](https://g Releasing --------- -Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/RELEASE_PROCESS.md). +Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/project/RELEASE-PROCESS.md). From 6571e079b9e7f1a67679c2cdc6322657f5030378 Mon Sep 17 00:00:00 2001 From: Per Persson Date: Thu, 15 Oct 2015 15:13:27 +0200 Subject: [PATCH 279/337] Remove incorrectly placed comment I'm not sure if it should be there at all, but at least it should hardly be where it currently is located. Signed-off-by: Per Persson --- docs/compose-file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 67322335a7..90730fecd0 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -48,8 +48,6 @@ See `man 7 capabilities` for a full list. - NET_ADMIN - SYS_ADMIN -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. - ### command Override the default command. @@ -106,6 +104,8 @@ Compose will use an alternate file to build with. dockerfile: Dockerfile-alternate +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + ### env_file Add environment variables from a file. Can be a single value or a list. From 558098d322131e100db52382b7826d02882efc4f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:14:04 -0400 Subject: [PATCH 280/337] Add a script to generate contributor list. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 11 +++++++---- script/release/contributors | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100755 script/release/contributors diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 85bbaf2950..a7fea69edf 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -82,17 +82,20 @@ When prompted build the non-linux binaries and test them. 5. Attach the binaries and `script/run.sh` -6. If everything looks good, it's time to push the release. +6. Add "Thanks" with a list of contributors. The contributor list can be generated + by running `./script/release/contributors`. + +7. If everything looks good, it's time to push the release. ./script/release/push-release -7. Publish the release on GitHub. +8. Publish the release on GitHub. -8. Check that both binaries download (following the install instructions) and run. +9. Check that both binaries download (following the install instructions) and run. -9. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) diff --git a/script/release/contributors b/script/release/contributors new file mode 100755 index 0000000000..bb9fe871ca --- /dev/null +++ b/script/release/contributors @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + + +function usage() { + >&2 cat << EOM +Print the list of github contributors for the release + +Usage: + + $0 +EOM + exit 1 +} + +[[ -n "$1" ]] || usage +PREV_RELEASE=$1 +VERSION=HEAD +URL="https://api.github.com/repos/docker/compose/compare" + +curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ + jq -r '.commits[].author.login' | \ + sort | \ + uniq -c | \ + sort -nr | \ + awk '{print "@"$2","}' | \ + xargs echo From b2f9c182f3029b087aee3f027f80254eea9ca284 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:16:58 -0400 Subject: [PATCH 281/337] Fix some release docs. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index a7fea69edf..ffa18077f4 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -54,7 +54,7 @@ When prompted build the non-linux binaries and test them. 2. Download the windows binary from AppVeyor - https://ci.appveyor.com/project/docker/compose/build//artifacts + https://ci.appveyor.com/project/docker/compose 3. Draft a release from the tag on GitHub (the script will open the window for you) @@ -93,7 +93,7 @@ When prompted build the non-linux binaries and test them. 8. Publish the release on GitHub. -9. Check that both binaries download (following the install instructions) and run. +9. Check that all the binaries download (following the install instructions) and run. 10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. From 284cda087e822fac205396aaacd135eb30ea60c3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:20:57 -0400 Subject: [PATCH 282/337] Add missing merge for release branch. Signed-off-by: Daniel Nephin --- script/release/make-branch | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/release/make-branch b/script/release/make-branch index dde1fb65de..e2eae4d5f2 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -39,7 +39,7 @@ fi DEFAULT_REMOTE=release REMOTE="$(find_remote "$GITHUB_REPO")" -# If we don't have a docker origin add one +# If we don't have a docker remote add one if [ -z "$REMOTE" ]; then echo "Creating $DEFAULT_REMOTE remote" git remote add ${DEFAULT_REMOTE} ${GITHUB_REPO} @@ -55,6 +55,8 @@ read -n1 -r -p "Continue? (ctrl+c to cancel)" git fetch $REMOTE -p git checkout -b $BRANCH $BASE_VERSION +echo "Merging remote release branch into new release branch" +git merge --strategy=ours --no-edit $REMOTE/release # Store the release version for this branch in git, so that other release # scripts can use it From 883f251e7d3f26e3fc5cc8658edde3fdbe6c97a1 Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Wed, 14 Oct 2015 19:06:05 +0100 Subject: [PATCH 283/337] Docs for shorthand notation of extends. Issue #1989 Signed-off-by: Karol Duleba --- docs/extends.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/extends.md b/docs/extends.md index d88ce61c5d..f0b9e9ea2d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -59,6 +59,10 @@ You can go further and define (or re-define) configuration locally in - DEBUG=1 cpu_shares: 5 + important_web: + extends: web + cpu_shares: 10 + You can also write other services and link your `web` service to them: web: @@ -233,7 +237,8 @@ manually keep both environments in sync. ### Reference You can use `extends` on any service together with other configuration keys. It -always expects a dictionary that should always contain the key: `service` and optionally the `file` key. +expects a dictionary that contains a `service` key and optionally a `file` key. +The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. The `file` key specifies the location of a Compose configuration file defining the extension. The `file` value can be an absolute or relative path. If you From 46de4411a7a4591874cfc41241e9393cc4826ea0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 16 Oct 2015 11:58:27 +0530 Subject: [PATCH 284/337] Revert networking-related changes to getting started guides Signed-off-by: Aanand Prasad --- docs/django.md | 7 +++++-- docs/rails.md | 4 +++- docs/wordpress.md | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/django.md b/docs/django.md index 2ebf4b4b97..c7ebf58bfe 100644 --- a/docs/django.md +++ b/docs/django.md @@ -64,8 +64,9 @@ and a `docker-compose.yml` file. The `docker-compose.yml` file describes the services that make your app. In this example those services are a web server and database. The compose file - also describes which Docker images these services use, any volumes they might - need mounted inside the containers, and any ports they might + also describes which Docker images these services use, how they link + together, any volumes they might need mounted inside the containers. + Finally, the `docker-compose.yml` file describes which ports these services expose. See the [`docker-compose.yml` reference](yml.md) for more information on how this file works. @@ -80,6 +81,8 @@ and a `docker-compose.yml` file. - .:/code ports: - "8000:8000" + links: + - db This file defines two services: The `db` service and the `web` service. diff --git a/docs/rails.md b/docs/rails.md index 9801ef7419..a33cac26ed 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,7 +37,7 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port. +Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. db: image: postgres @@ -48,6 +48,8 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th - .:/myapp ports: - "3000:3000" + links: + - db ### Build the project diff --git a/docs/wordpress.md b/docs/wordpress.md index 5c9bcdbd9f..8c1f5b0acb 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -46,6 +46,8 @@ and a separate MySQL instance: command: php -S 0.0.0.0:8000 -t /code ports: - "8000:8000" + links: + - db volumes: - .:/code db: From 6048630a1194d439a698050aa423fb774bf7773c Mon Sep 17 00:00:00 2001 From: Cameron Eagans Date: Thu, 15 Oct 2015 16:58:27 -0600 Subject: [PATCH 285/337] docker-compose pull SERVICE should not pull SERVICE's dependencies Signed-off-by: Cameron Eagans --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 0e20a4cee7..fdd70caf11 100644 --- a/compose/project.py +++ b/compose/project.py @@ -335,7 +335,7 @@ class Project(object): return plans def pull(self, service_names=None, ignore_pull_failures=False): - for service in self.get_services(service_names, include_deps=True): + for service in self.get_services(service_names, include_deps=False): service.pull(ignore_pull_failures) def containers(self, service_names=None, stopped=False, one_off=False): From 49b98fa111bcea0086c35d91d0e42389139906a6 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 16 Oct 2015 12:57:54 +0100 Subject: [PATCH 286/337] Attempt to document escaping env vars People are likely to run into their env vars being set to empty strings, if they're not aware that they need to escape them for Compose to not interpolate them. Signed-off-by: Mazz Mosley --- docs/compose-file.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 90730fecd0..b72a7cc437 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -428,9 +428,18 @@ Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not supported. -If you need to put a literal dollar sign in a configuration value, use a double -dollar sign (`$$`). +You can use a `$$` (double-dollar sign) when your configuration needs a literal +dollar sign. This also prevents Compose from interpolating a value, so a `$$` +allows you to refer to environment variables that you don't want processed by +Compose. + web: + build: . + command: "$$VAR_NOT_INTERPOLATED_BY_COMPOSE" + +If you forget and use a single dollar sign (`$`), Compose interprets the value as an environment variable and will warn you: + + The VAR_NOT_INTERPOLATED_BY_COMPOSE is not set. Substituting an empty string. ## Compose documentation From 6f45eb795976c6a4a8b7014b250ecc583b121d40 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sat, 17 Oct 2015 09:26:32 -0700 Subject: [PATCH 287/337] bash completion for networking options Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e64b24a009..0eed1f18b7 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -105,11 +105,15 @@ _docker_compose_docker_compose() { --project-name|-p) return ;; + --x-network-driver) + COMPREPLY=( $( compgen -W "bridge host none overlay" -- "$cur" ) ) + return + ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -h --verbose --version -v --file -f --project-name -p" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v --x-networking --x-network-driver" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -410,6 +414,9 @@ _docker_compose() { (( counter++ )) compose_project="${words[$counter]}" ;; + --x-network-driver) + (( counter++ )) + ;; -*) ;; *) From 20d34c8b14113eb430c0d350c23f0088b6cd8047 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 15 Oct 2015 23:02:40 +0200 Subject: [PATCH 288/337] Add zsh completion for 'docker-compose --x-networking --x-network-driver' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index cefcb109e2..d79b25d165 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -330,6 +330,8 @@ _docker-compose() { '(- :)'{-v,--version}'[Print version and exit]' \ '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '--x-networking[(EXPERIMENTAL) Use new Docker networking functionality. Requires Docker 1.9 or later.]' \ + '--x-network-driver[(EXPERIMENTAL) Specify a network driver (default: "bridge"). Requires Docker 1.9 or later.]:Network Driver:(bridge host none overlay)' \ '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 From 514f0650b2a857d6516d6ceb69aba6335d5618f2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 14:47:04 -0400 Subject: [PATCH 289/337] Give the user a better error message (without a stack trace) when there is a yaml error. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 +++-- tests/unit/config/config_test.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3bcd769ad5..59b98f6092 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -610,5 +610,6 @@ def load_yaml(filename): try: with open(filename, 'r') as fh: return yaml.safe_load(fh) - except IOError as e: - raise ConfigurationError(six.text_type(e)) + except (IOError, yaml.YAMLError) as e: + error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ + raise ConfigurationError(u"{}: {}".format(error_name, e)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 20ae7fa305..b4bd9c7123 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5,6 +5,7 @@ import shutil import tempfile from operator import itemgetter +import py import pytest from compose.config import config @@ -349,6 +350,18 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_yaml_with_yaml_error(self): + tmpdir = py.test.ensuretemp('invalid_yaml_test') + invalid_yaml_file = tmpdir.join('docker-compose.yml') + invalid_yaml_file.write(""" + web: + this is bogus: ok: what + """) + with pytest.raises(ConfigurationError) as exc: + config.load_yaml(str(invalid_yaml_file)) + + assert 'line 3, column 32' in exc.exconly() + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 24d4a1045a6317a612f856af3cc99e1b301eae49 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 15:45:56 -0400 Subject: [PATCH 290/337] Fixes #2203 - properly validate files when multiple files are used. Remove the single-use decorators so the functionality can be used directly as a function. Signed-off-by: Daniel Nephin --- compose/config/config.py | 22 +++++++------------- compose/config/validation.py | 35 +++++++++++++------------------- tests/unit/config/config_test.py | 22 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 59b98f6092..05e5725857 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,7 +14,6 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extended_service_exists from .validation import validate_extends_file_path -from .validation import validate_service_names from .validation import validate_top_level_object @@ -165,16 +164,6 @@ def find_candidates_in_parent_dirs(filenames, path): return (candidates, path) -@validate_top_level_object -@validate_service_names -def pre_process_config(config): - """ - Pre validation checks and processing of the config file to interpolate env - vars returning a config dict ready to be tested against the schema. - """ - return interpolate_environment_variables(config) - - def load(config_details): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top @@ -194,7 +183,7 @@ def load(config_details): return service_dict def load_file(filename, config): - processed_config = pre_process_config(config) + processed_config = interpolate_environment_variables(config) validate_against_fields_schema(processed_config) return [ build_service(filename, name, service_config) @@ -209,7 +198,10 @@ def load(config_details): } config_file = config_details.config_files[0] + validate_top_level_object(config_file.config) for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file.config) + config_file = ConfigFile( config_file.filename, merge_services(config_file.config, next_file.config)) @@ -283,9 +275,9 @@ class ServiceLoader(object): ) self.extended_service_name = extends['service'] - full_extended_config = pre_process_config( - load_yaml(self.extended_config_path) - ) + config = load_yaml(self.extended_config_path) + validate_top_level_object(config) + full_extended_config = interpolate_environment_variables(config) validate_extended_service_exists( self.extended_service_name, diff --git a/compose/config/validation.py b/compose/config/validation.py index 46b891b71b..aea722864c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -2,7 +2,6 @@ import json import logging import os import sys -from functools import wraps import six from docker.utils.ports import split_port @@ -65,27 +64,21 @@ def format_boolean_in_environment(instance): return True -def validate_service_names(func): - @wraps(func) - def func_wrapper(config): - for service_name in config.keys(): - if type(service_name) is int: - raise ConfigurationError( - "Service name: {} needs to be a string, eg '{}'".format(service_name, service_name) - ) - return func(config) - return func_wrapper - - -def validate_top_level_object(func): - @wraps(func) - def func_wrapper(config): - if not isinstance(config, dict): +def validate_service_names(config): + for service_name in config.keys(): + if not isinstance(service_name, six.string_types): raise ConfigurationError( - "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - ) - return func(config) - return func_wrapper + "Service name: {} needs to be a string, eg '{}'".format( + service_name, + service_name)) + + +def validate_top_level_object(config): + if not isinstance(config, dict): + raise ConfigurationError( + "Top level object needs to be a dictionary. Check your .yml file " + "that you have defined a service at the top level.") + validate_service_names(config) def validate_extends_file_path(service_name, extends_options, filename): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b4bd9c7123..e8caea814d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -134,6 +134,28 @@ class ConfigTest(unittest.TestCase): ] self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_load_with_multiple_files_and_empty_override(self): + base_file = config.ConfigFile( + 'base.yaml', + {'web': {'image': 'example/web'}}) + override_file = config.ConfigFile('override.yaml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + + def test_load_with_multiple_files_and_empty_base(self): + base_file = config.ConfigFile('base.yaml', None) + override_file = config.ConfigFile( + 'override.yaml', + {'web': {'image': 'example/web'}}) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From 5fdb75b541cee7a6a26e7b9e5a6483afebc88174 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 19 Oct 2015 14:36:56 +0100 Subject: [PATCH 291/337] Improve error message for type constraints It was missing a space between the different types, when there were 3 possible type values. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index aea722864c..b21e12cb84 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -136,7 +136,7 @@ def process_errors(errors, service_name=None): if len(validator) >= 2: first_type = anglicize_validator(validator[0]) last_type = anglicize_validator(validator[-1]) - types_from_validator = "{}{}".format(first_type, ", ".join(validator[1:-1])) + types_from_validator = "{}".format(", ".join([first_type] + validator[1:-1])) msg = "{} or {}".format( types_from_validator, @@ -156,7 +156,6 @@ def process_errors(errors, service_name=None): Inspecting the context value of a ValidationError gives us information about which sub schema failed and which kind of error it is. """ - required = [context for context in error.context if context.validator == 'required'] if required: return required[0].message From 0e4f9c9a66f844cf725870c21cb5b5318b28ce16 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 19 Oct 2015 15:15:24 +0100 Subject: [PATCH 292/337] Environment keys can contain empty values Environment keys that contain no value, get populated with values taken from the environment not from the build phase but from running the command `up`. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 2 +- compose/config/validation.py | 2 +- tests/unit/config/config_test.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index cc37f444de..e254e3539f 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -41,7 +41,7 @@ "type": "object", "patternProperties": { "^[^-]+$": { - "type": ["string", "number", "boolean"], + "type": ["string", "number", "boolean", "null"], "format": "environment" } }, diff --git a/compose/config/validation.py b/compose/config/validation.py index b21e12cb84..8cfc405fe8 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -136,7 +136,7 @@ def process_errors(errors, service_name=None): if len(validator) >= 2: first_type = anglicize_validator(validator[0]) last_type = anglicize_validator(validator[-1]) - types_from_validator = "{}".format(", ".join([first_type] + validator[1:-1])) + types_from_validator = ", ".join([first_type] + validator[1:-1]) msg = "{} or {}".format( types_from_validator, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e8caea814d..e32e5b47c1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -449,6 +449,23 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) + def test_empty_environment_key_allowed(self): + service_dict = config.load( + build_config_details( + { + 'web': { + 'build': '.', + 'environment': { + 'POSTGRES_PASSWORD': '' + }, + }, + }, + '.', + None, + ) + )[0] + self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') + class VolumeConfigTest(unittest.TestCase): def test_no_binding(self): From bf672ec3405c31a4c9d64984876665c68128c7fe Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 11:18:07 -0400 Subject: [PATCH 293/337] Fixes #2205 - extends must be copied from override file. Signed-off-by: Daniel Nephin --- compose/config/config.py | 19 ++++++++++++---- tests/unit/config/config_test.py | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 05e5725857..ff8861b51a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -193,7 +193,9 @@ def load(config_details): def merge_services(base, override): all_service_names = set(base) | set(override) return { - name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + name: merge_service_dicts_from_files( + base.get(name, {}), + override.get(name, {})) for name in all_service_names } @@ -270,9 +272,7 @@ class ServiceLoader(object): extends, self.filename ) - self.extended_config_path = self.get_extended_config_path( - extends - ) + self.extended_config_path = self.get_extended_config_path(extends) self.extended_service_name = extends['service'] config = load_yaml(self.extended_config_path) @@ -355,6 +355,17 @@ def process_container_options(service_dict, working_dir=None): return service_dict +def merge_service_dicts_from_files(base, override): + """When merging services from multiple files we need to merge the `extends` + field. This is not handled by `merge_service_dicts()` which is used to + perform the `extends`. + """ + new_service = merge_service_dicts(base, override) + if 'extends' in override: + new_service['extends'] = override['extends'] + return new_service + + def merge_service_dicts(base, override): d = base.copy() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e32e5b47c1..c8b76f36d0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -156,6 +156,43 @@ class ConfigTest(unittest.TestCase): config.load(details) assert 'Top level object needs to be a dictionary' in exc.exconly() + def test_load_with_multiple_files_and_extends_in_override_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': {'image': 'example/web'}, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'web': { + 'extends': { + 'file': 'common.yml', + 'service': 'base', + }, + 'volumes': ['/home/user/project:/code'], + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + tmpdir = py.test.ensuretemp('config_test') + tmpdir.join('common.yml').write(""" + base: + labels: ['label=one'] + """) + with tmpdir.as_cwd(): + service_dicts = config.load(details) + + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'volumes': ['/home/user/project:/code'], + 'labels': {'label': 'one'}, + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From 725088a18b81f0c1451006b5ea4999cae641aba6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 11:39:06 -0400 Subject: [PATCH 294/337] Force windows drives to be lowercase. Signed-off-by: Daniel Nephin --- compose/service.py | 2 +- tests/unit/config/config_test.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5f1d59468c..7daf7f2f9c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -952,7 +952,7 @@ def normalize_paths_for_engine(external_path, internal_path): drive, tail = os.path.splitdrive(external_path) if drive: - reformatted_drive = "/{}".format(drive.replace(":", "")) + reformatted_drive = "/{}".format(drive.lower().replace(":", "")) external_path = reformatted_drive + tail external_path = "/".join(external_path.split("\\")) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c8b76f36d0..d15cd9a689 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -537,8 +537,8 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/var/lib/data:/data']) def test_absolute_windows_path_does_not_expand(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['C:\\data:/data']}, working_dir='.') - self.assertEqual(d['volumes'], ['C:\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['c:\\data:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['c:\\data:/data']) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') def test_relative_path_does_expand_posix(self): @@ -553,14 +553,14 @@ class VolumeConfigTest(unittest.TestCase): @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') def test_relative_path_does_expand_windows(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject\\data:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\otherproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) @mock.patch.dict(os.environ) def test_home_directory_with_driver_does_not_expand(self): From f290faf4ba8e4b98126909a4181534ff1fa20f30 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 11:49:10 -0400 Subject: [PATCH 295/337] Minor refactor to use guard and replace instead of split+join Signed-off-by: Daniel Nephin --- compose/service.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/compose/service.py b/compose/service.py index 7daf7f2f9c..f18afa485b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -943,24 +943,22 @@ def build_volume_binding(volume_spec): def normalize_paths_for_engine(external_path, internal_path): - """ - Windows paths, c:\my\path\shiny, need to be changed to be compatible with + """Windows paths, c:\my\path\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ - if IS_WINDOWS_PLATFORM: - if external_path: - drive, tail = os.path.splitdrive(external_path) - - if drive: - reformatted_drive = "/{}".format(drive.lower().replace(":", "")) - external_path = reformatted_drive + tail - - external_path = "/".join(external_path.split("\\")) - - return external_path, "/".join(internal_path.split("\\")) - else: + if not IS_WINDOWS_PLATFORM: return external_path, internal_path + if external_path: + drive, tail = os.path.splitdrive(external_path) + + if drive: + external_path = '/' + drive.lower().rstrip(':') + tail + + external_path = external_path.replace('\\', '/') + + return external_path, internal_path.replace('\\', '/') + def parse_volume_spec(volume_config): """ From f5ad36314387e2b97c2f1be3303da50ff5e4cfdb Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 13:05:14 -0400 Subject: [PATCH 296/337] Use inspect network to query for an existing network. And more tests for get_network() Signed-off-by: Daniel Nephin --- compose/project.py | 9 +++++---- tests/integration/project_test.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/project.py b/compose/project.py index fdd70caf11..d4934c268a 100644 --- a/compose/project.py +++ b/compose/project.py @@ -5,6 +5,7 @@ import logging from functools import reduce from docker.errors import APIError +from docker.errors import NotFound from .config import ConfigurationError from .config import get_service_name_from_net @@ -363,10 +364,10 @@ class Project(object): return [c for c in containers if matches_service_names(c)] def get_network(self): - networks = self.client.networks(names=[self.name]) - if networks: - return networks[0] - return None + try: + return self.client.inspect_network(self.name) + except NotFound: + return None def ensure_network_exists(self): # TODO: recreate network if driver has changed? diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ff50c80b2a..ac0f121cf2 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase +from compose.cli.docker_client import docker_client from compose.config import config from compose.const import LABEL_PROJECT from compose.container import Container @@ -96,6 +97,22 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) + def test_get_network_does_not_exist(self): + self.require_engine_version("1.9") + client = docker_client(version='1.21') + + project = Project('composetest', [], client) + assert project.get_network() is None + + def test_get_network(self): + self.require_engine_version("1.9") + client = docker_client(version='1.21') + + network_name = 'network_does_exist' + project = Project(network_name, [], client) + client.create_network(network_name) + assert project.get_network()['name'] == network_name + def test_net_from_service(self): project = Project.from_dicts( name='composetest', From 95a23eb6829a48298f7db7fb806fc944f0c6cc18 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 15:40:50 -0400 Subject: [PATCH 297/337] Change version check from engine version to api version. Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 4 ++-- tests/integration/project_test.py | 4 ++-- tests/integration/testcases.py | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 78519d1418..19cc822ee1 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -187,7 +187,7 @@ class CLITestCase(DockerClientTestCase): ) def test_up_without_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d'], None) @@ -205,7 +205,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(web_container.get('HostConfig.Links')) def test_up_with_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['--x-networking', 'up', '-d'], None) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ac0f121cf2..fd45b9393f 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -98,14 +98,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) def test_get_network_does_not_exist(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') client = docker_client(version='1.21') project = Project('composetest', [], client) assert project.get_network() is None def test_get_network(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') client = docker_client(version='1.21') network_name = 'network_does_exist' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a412fb04fb..686a2b69a4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -76,11 +76,7 @@ class DockerClientTestCase(unittest.TestCase): build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) - def require_engine_version(self, minimum): - # Drop '-dev' or '-rcN' suffix - engine = self.client.version()['Version'].split('-', 1)[0] - if version_lt(engine, minimum): - skip( - "Engine version is too low ({} < {})" - .format(engine, minimum) - ) + def require_api_version(self, minimum): + api_version = self.client.version()['ApiVersion'] + if version_lt(api_version, minimum): + skip("API version is too low ({} < {})".format(api_version, minimum)) From e168fd03ca9ea4cdb2843f706b28a864ae669174 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 22 Oct 2015 12:12:43 -0400 Subject: [PATCH 298/337] Fix unicode in environment variables for python2. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 ++++- tests/fixtures/env/resolve.env | 2 +- tests/unit/cli_test.py | 5 +++-- tests/unit/config/config_test.py | 8 +++++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ff8861b51a..21549e9b34 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,3 +1,4 @@ +import codecs import logging import os import sys @@ -451,6 +452,8 @@ def parse_environment(environment): def split_env(env): + if isinstance(env, six.binary_type): + env = env.decode('utf-8') if '=' in env: return env.split('=', 1) else: @@ -473,7 +476,7 @@ def env_vars_from_file(filename): if not os.path.exists(filename): raise ConfigurationError("Couldn't find env file: %s" % filename) env = {} - for line in open(filename, 'r'): + for line in codecs.open(filename, 'r', 'utf-8'): line = line.strip() if line and not line.startswith('#'): k, v = split_env(line) diff --git a/tests/fixtures/env/resolve.env b/tests/fixtures/env/resolve.env index 720520d29e..b4f76b29ed 100644 --- a/tests/fixtures/env/resolve.env +++ b/tests/fixtures/env/resolve.env @@ -1,4 +1,4 @@ -FILE_DEF=F1 +FILE_DEF=bär FILE_DEF_EMPTY= ENV_DEF NO_DEF diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 0c78e6bbfe..5b63d2e84a 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import absolute_import from __future__ import unicode_literals @@ -98,7 +99,7 @@ class CLITestCase(unittest.TestCase): command.run(mock_project, { 'SERVICE': 'service', 'COMMAND': None, - '-e': ['BAR=NEW', 'OTHER=THREE'], + '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], '--user': None, '--no-deps': None, '--allow-insecure-ssl': None, @@ -114,7 +115,7 @@ class CLITestCase(unittest.TestCase): _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertEqual( call_kwargs['environment'], - {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': 'THREE'}) + {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': u'bär'}) def test_run_service_with_restart_always(self): command = TopLevelCommand() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d15cd9a689..a54b006fa6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import print_function import os @@ -894,7 +895,12 @@ class EnvTest(unittest.TestCase): ) self.assertEqual( service_dict['environment'], - {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, + { + 'FILE_DEF': u'bär', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': 'E3', + 'NO_DEF': '' + }, ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') From 88e53e177dce3982f7106597bb0c431345a7099e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 16:42:43 -0400 Subject: [PATCH 299/337] Upgrade pyinstaller to 3.0 Signed-off-by: Daniel Nephin --- requirements-build.txt | 2 +- script/build-windows.ps1 | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/requirements-build.txt b/requirements-build.txt index 5da6fa4966..20aad4208c 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller +pyinstaller==3.0 diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 6e8a7c5ae7..42a4a501c1 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -42,11 +42,6 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -# TODO: pip warns when installing from a git sha, so we need to set ErrorAction to -# 'Continue'. See -# https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 -# This can be removed once pyinstaller 3.x is released and we upgrade -$ErrorActionPreference = "Continue" .\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . @@ -54,8 +49,9 @@ $ErrorActionPreference = "Continue" # Build binary # pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue +$ErrorActionPreference = "Continue" .\venv\Scripts\pyinstaller .\docker-compose.spec $ErrorActionPreference = "Stop" -Move-Item -Force .\dist\docker-compose .\dist\docker-compose-Windows-x86_64.exe +Move-Item -Force .\dist\docker-compose.exe .\dist\docker-compose-Windows-x86_64.exe .\dist\docker-compose-Windows-x86_64.exe --version From a9b4fe768d1a49a6b6fad9b36e1e41cd6be3b756 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 26 Oct 2015 13:29:59 -0400 Subject: [PATCH 300/337] Fix running one-off containers with --x-networking by disabling linking to self. docker create fails if networking and links are used together. Signed-off-by: Daniel Nephin --- compose/service.py | 3 +++ tests/unit/service_test.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/compose/service.py b/compose/service.py index f18afa485b..43067d42c4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -545,6 +545,9 @@ class Service(object): return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): + if self.use_networking: + return [] + links = [] for service, link_name in self.links: for container in service.containers(): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c5e1a9fb06..7149ff0eeb 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -444,6 +444,14 @@ class ServiceTest(unittest.TestCase): } self.assertEqual(config_dict, expected) + def test_get_links_with_networking(self): + service = Service( + 'foo', + image='foo', + links=[(Service('one'), 'one')], + use_networking=True) + self.assertEqual(service._get_links(link_to_self=True), []) + class NetTestCase(unittest.TestCase): From d6fa8596d22c6a330449ae808d6bb2ddbb5c2534 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 21 Oct 2015 17:28:16 +0100 Subject: [PATCH 301/337] Attach to a container's log_stream before they're started So we're not displaying output of all previous logs for a container, we attach, if possible, to a container before the container is started. LogPrinter checks if a container has a log_stream already attached and print from that rather than always attempting to attach one itself. Signed-off-by: Mazz Mosley --- compose/cli/log_printer.py | 10 ++++-- compose/cli/main.py | 11 +++++-- compose/container.py | 8 +++++ compose/project.py | 6 ++-- compose/service.py | 58 +++++++++++++++++++++------------ tests/integration/state_test.py | 4 ++- 6 files changed, 67 insertions(+), 30 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 6e1499e1d5..66920726ce 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -73,9 +73,13 @@ def build_no_log_generator(container, prefix, color_func): def build_log_generator(container, prefix, color_func): - # Attach to container before log printer starts running - stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) - line_generator = split_buffer(stream) + # if the container doesn't have a log_stream we need to attach to container + # before log printer starts running + if container.log_stream is None: + stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) + line_generator = split_buffer(stream) + else: + line_generator = split_buffer(container.log_stream) for line in line_generator: yield prefix + line diff --git a/compose/cli/main.py b/compose/cli/main.py index c800d95f98..5505b89f51 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -565,16 +565,18 @@ class TopLevelCommand(DocoptCommand): start_deps = not options['--no-deps'] service_names = options['SERVICE'] timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + detached = options.get('-d') to_attach = project.up( service_names=service_names, start_deps=start_deps, strategy=convergence_strategy_from_opts(options), do_build=not options['--no-build'], - timeout=timeout + timeout=timeout, + detached=detached ) - if not options['-d']: + if not detached: log_printer = build_log_printer(to_attach, service_names, monochrome) attach_to_logs(project, log_printer, service_names, timeout) @@ -636,7 +638,10 @@ def convergence_strategy_from_opts(options): def build_log_printer(containers, service_names, monochrome): if service_names: - containers = [c for c in containers if c.service in service_names] + containers = [ + container + for container in containers if container.service in service_names + ] return LogPrinter(containers, monochrome=monochrome) diff --git a/compose/container.py b/compose/container.py index a03acf56fd..64773b9e6a 100644 --- a/compose/container.py +++ b/compose/container.py @@ -19,6 +19,7 @@ class Container(object): self.client = client self.dictionary = dictionary self.has_been_inspected = has_been_inspected + self.log_stream = None @classmethod def from_ps(cls, client, dictionary, **kwargs): @@ -146,6 +147,13 @@ class Container(object): log_type = self.log_driver return not log_type or log_type == 'json-file' + def attach_log_stream(self): + """A log stream can only be attached if the container uses a json-file + log driver. + """ + if self.has_api_logs: + self.log_stream = self.attach(stdout=True, stderr=True, stream=True) + def get(self, key): """Return a value from the container or None if the value is not set. diff --git a/compose/project.py b/compose/project.py index d4934c268a..68edaddcb2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -290,7 +290,8 @@ class Project(object): start_deps=True, strategy=ConvergenceStrategy.changed, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): services = self.get_services(service_names, include_deps=start_deps) @@ -308,7 +309,8 @@ class Project(object): for container in service.execute_convergence_plan( plans[service.name], do_build=do_build, - timeout=timeout + timeout=timeout, + detached=detached ) ] diff --git a/compose/service.py b/compose/service.py index 43067d42c4..90ef709ec6 100644 --- a/compose/service.py +++ b/compose/service.py @@ -399,13 +399,17 @@ class Service(object): def execute_convergence_plan(self, plan, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): (action, containers) = plan + should_attach_logs = not detached if action == 'create': - container = self.create_container( - do_build=do_build, - ) + container = self.create_container(do_build=do_build) + + if should_attach_logs: + container.attach_log_stream() + self.start_container(container) return [container] @@ -413,15 +417,16 @@ class Service(object): elif action == 'recreate': return [ self.recreate_container( - c, - timeout=timeout + container, + timeout=timeout, + attach_logs=should_attach_logs ) - for c in containers + for container in containers ] elif action == 'start': - for c in containers: - self.start_container_if_stopped(c) + for container in containers: + self.start_container_if_stopped(container, attach_logs=should_attach_logs) return containers @@ -434,16 +439,7 @@ class Service(object): else: raise Exception("Invalid action: {}".format(action)) - def recreate_container(self, - container, - timeout=DEFAULT_TIMEOUT): - """Recreate a container. - - The original container is renamed to a temporary name so that data - volumes can be copied to the new container, before the original - container is removed. - """ - log.info("Recreating %s" % container.name) + def _recreate_stop_container(self, container, timeout): try: container.stop(timeout=timeout) except APIError as e: @@ -454,26 +450,46 @@ class Service(object): else: raise + def _recreate_rename_container(self, container): # Use a hopefully unique container name by prepending the short id self.client.rename( container.id, - '%s_%s' % (container.short_id, container.name)) + '%s_%s' % (container.short_id, container.name) + ) + def recreate_container(self, + container, + timeout=DEFAULT_TIMEOUT, + attach_logs=False): + """Recreate a container. + + The original container is renamed to a temporary name so that data + volumes can be copied to the new container, before the original + container is removed. + """ + log.info("Recreating %s" % container.name) + + self._recreate_stop_container(container, timeout) + self._recreate_rename_container(container) new_container = self.create_container( do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, ) + if attach_logs: + new_container.attach_log_stream() self.start_container(new_container) container.remove() return new_container - def start_container_if_stopped(self, container): + def start_container_if_stopped(self, container, attach_logs=False): if container.is_running: return container else: log.info("Starting %s" % container.name) + if attach_logs: + container.attach_log_stream() return self.start_container(container) def start_container(self, container): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index ef7276bd8d..02e9d31526 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -18,6 +18,7 @@ from compose.service import ConvergenceStrategy class ProjectTestCase(DockerClientTestCase): def run_up(self, cfg, **kwargs): kwargs.setdefault('timeout', 1) + kwargs.setdefault('detached', True) project = self.make_project(cfg) project.up(**kwargs) @@ -184,7 +185,8 @@ def converge(service, do_build=True): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) + containers, logging_threads = zip(*service.execute_convergence_plan(plan, do_build=do_build, timeout=1)) + return containers class ServiceStateTest(DockerClientTestCase): From da41ed22f9358069daab28cb08ac55e0a31e4816 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 26 Oct 2015 10:27:57 +0000 Subject: [PATCH 302/337] Fix tests Signed-off-by: Mazz Mosley --- tests/integration/service_test.py | 1 + tests/integration/state_test.py | 3 +-- tests/unit/cli/log_printer_test.py | 1 + tests/unit/service_test.py | 4 +--- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8a8e4d54d9..38d7d5b55b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -362,6 +362,7 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) + self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 02e9d31526..3230aefc61 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -185,8 +185,7 @@ def converge(service, do_build=True): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - containers, logging_threads = zip(*service.execute_convergence_plan(plan, do_build=do_build, timeout=1)) - return containers + return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 2c91689807..575fcaf7b5 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -16,6 +16,7 @@ def build_mock_container(reader): name='myapp_web_1', name_without_project='web_1', has_api_logs=True, + log_stream=None, attach=reader, wait=mock.Mock(return_value=0), ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7149ff0eeb..d86f80f730 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -323,9 +323,7 @@ class ServiceTest(unittest.TestCase): new_container = service.recreate_container(mock_container) mock_container.stop.assert_called_once_with(timeout=10) - self.mock_client.rename.assert_called_once_with( - mock_container.id, - '%s_%s' % (mock_container.short_id, mock_container.name)) + mock_container.rename_to_tmp_name.assert_called_once_with() new_container.start.assert_called_once_with() mock_container.remove.assert_called_once_with() From 6f0096c87b6a33895752f1a5f6b914e40c2ceee1 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 27 Oct 2015 11:55:35 +0000 Subject: [PATCH 303/337] Move rename functionality into Container Signed-off-by: Mazz Mosley --- compose/container.py | 9 +++++++++ compose/service.py | 9 +-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/compose/container.py b/compose/container.py index 64773b9e6a..dd69e8dddb 100644 --- a/compose/container.py +++ b/compose/container.py @@ -192,6 +192,15 @@ class Container(object): def remove(self, **options): return self.client.remove_container(self.id, **options) + def rename_to_tmp_name(self): + """Rename the container to a hopefully unique temporary container name + by prepending the short id. + """ + self.client.rename( + self.id, + '%s_%s' % (self.short_id, self.name) + ) + def inspect_if_not_inspected(self): if not self.has_been_inspected: self.inspect() diff --git a/compose/service.py b/compose/service.py index 90ef709ec6..ac0a84906b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -450,13 +450,6 @@ class Service(object): else: raise - def _recreate_rename_container(self, container): - # Use a hopefully unique container name by prepending the short id - self.client.rename( - container.id, - '%s_%s' % (container.short_id, container.name) - ) - def recreate_container(self, container, timeout=DEFAULT_TIMEOUT, @@ -470,7 +463,7 @@ class Service(object): log.info("Recreating %s" % container.name) self._recreate_stop_container(container, timeout) - self._recreate_rename_container(container) + container.rename_to_tmp_name() new_container = self.create_container( do_build=False, previous_container=container, From a772a0d7d7c454fe35cb65b40a616df5413555c8 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 27 Oct 2015 11:59:09 +0000 Subject: [PATCH 304/337] Remove redundant try/except Code cleanup. We no longer need this as the api returns a 304 for any stopped containers, which doesn't raise an error. Signed-off-by: Mazz Mosley --- compose/service.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/compose/service.py b/compose/service.py index ac0a84906b..dbcec10382 100644 --- a/compose/service.py +++ b/compose/service.py @@ -439,17 +439,6 @@ class Service(object): else: raise Exception("Invalid action: {}".format(action)) - def _recreate_stop_container(self, container, timeout): - try: - container.stop(timeout=timeout) - except APIError as e: - if (e.response.status_code == 500 - and e.explanation - and 'no such process' in str(e.explanation)): - pass - else: - raise - def recreate_container(self, container, timeout=DEFAULT_TIMEOUT, @@ -462,7 +451,7 @@ class Service(object): """ log.info("Recreating %s" % container.name) - self._recreate_stop_container(container, timeout) + container.stop(timeout=timeout) container.rename_to_tmp_name() new_container = self.create_container( do_build=False, From 29b0ffe5e99be0c49e750eb29b938972449c5bcc Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 22 Oct 2015 17:17:11 +1000 Subject: [PATCH 305/337] Possible link fixes Signed-off-by: Sven Dowideit --- docs/django.md | 2 +- docs/env.md | 2 +- docs/index.md | 2 +- docs/networking.md | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/django.md b/docs/django.md index c7ebf58bfe..fd18784ecc 100644 --- a/docs/django.md +++ b/docs/django.md @@ -67,7 +67,7 @@ and a `docker-compose.yml` file. also describes which Docker images these services use, how they link together, any volumes they might need mounted inside the containers. Finally, the `docker-compose.yml` file describes which ports these services - expose. See the [`docker-compose.yml` reference](yml.md) for more + expose. See the [`docker-compose.yml` reference](compose-file.md) for more information on how this file works. 9. Add the following configuration to the file. diff --git a/docs/env.md b/docs/env.md index 8f3cc3ccb1..d7b51ba2b5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -11,7 +11,7 @@ weight=3 # Compose environment variables reference -**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](yml.md#links) for details. +**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. diff --git a/docs/index.md b/docs/index.md index e19e7d7f44..62c78d6893 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,7 +154,7 @@ Now, when you run `docker-compose up`, Compose will pull a Redis image, build an If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try http://localhost:5000. +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. You should get a message in your browser saying: diff --git a/docs/networking.md b/docs/networking.md index f4227917ac..9a6d792df4 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,11 +12,11 @@ weight=6 # Networking in Compose -> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. +> **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. -Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. +Compose sets up a single default [network](/engine/reference/commandline/network_create.md) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. -> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it. +> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: @@ -65,7 +65,7 @@ Docker links are a one-way, single-host communication system. They should now be ## Specifying the network driver -By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs). +By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](/engine/extend/plugins_network.md). You can specify which one to use with the `--x-network-driver` flag: From 8cc8e614740a71234cb7d1d3d5456c2c316a022c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 12:55:32 -0400 Subject: [PATCH 306/337] Bump 1.5.0rc2 Signed-off-by: Daniel Nephin Fill out 1.5.0 release notes Signed-off-by: Aanand Prasad --- CHANGELOG.md | 102 ++++++++++++++++++++++++++++++-------------- compose/__init__.py | 2 +- docs/install.md | 4 +- script/run.sh | 2 +- 4 files changed, 75 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730cd30ef7..3b2ecd97e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,48 +4,88 @@ Change log 1.5.0 (2015-10-13) ------------------ -Major Features +Major features: -- Compose is now available on windows. -- Environment variable can be used in the compose file. See +- Compose is now available for Windows. + +- Environment variables can be used in the Compose file. See https://github.com/docker/compose/blob/129092b7/docs/yml.md#variable-substitution + - Multiple compose files can be specified, allowing you to override - setting in the default compose file. See + settings in the default Compose file. See https://github.com/docker/compose/blob/129092b7/docs/reference/docker-compose.md for more details. -- Configuration validation is now a lot more strict -- `up` now waits for all services to exit before shutting down -- Support for the new docker networking can be enabled with - the `--x-networking` flag -New Features +- Compose now produces better error messages when a file contains + invalid configuration. -- `volumes_from` now supports a mode option allowing for read-only - `volumes_from` -- Volumes that don't start with a path indicator (`.` or `/`) will now be - treated as a named volume. Previously this was a warning. -- `--pull` flag added to `build` -- `--ignore-pull-failures` flag added to `pull` -- Support for the `ipc` field added to the compose file -- Containers created by `run` can now be named with the `--name` flag -- If you install Compose with pip or use it as a library, it now - works with Python 3 -- `image` field now supports image digests (in addition to ids and tags) -- `ports` now supports ranges of ports -- `--publish` flag added to `run` -- New subcommands `pause` and `unpause` -- services may be extended from the same file without a `file` key in - `extends` -- Compose can be installed and run as a docker image. This is an experimental +- `up` now waits for all services to exit before shutting down, + rather than shutting down as soon as one container exits. + +- Experimental support for the new docker networking system can be + enabled with the `--x-networking` flag. Read more here: + https://github.com/docker/docker/blob/8fee1c20/docs/userguide/dockernetworks.md + +New features: + +- You can now optionally pass a mode to `volumes_from`, e.g. + `volumes_from: ["servicename:ro"]`. + +- Since Docker now lets you create volumes with names, you can refer to those + volumes by name in `docker-compose.yml`. For example, + `volumes: ["mydatavolume:/data"]` will mount the volume named + `mydatavolume` at the path `/data` inside the container. + + If the first component of an entry in `volumes` starts with a `.`, `/` or + `~`, it is treated as a path and expansion of relative paths is performed as + necessary. Otherwise, it is treated as a volume name and passed straight + through to Docker. + + Read more on named volumes and volume drivers here: + https://github.com/docker/docker/blob/244d9c33/docs/userguide/dockervolumes.md + +- `docker-compose build --pull` instructs Compose to pull the base image for + each Dockerfile before building. + +- `docker-compose pull --ignore-pull-failures` instructs Compose to continue + if it fails to pull a single service's image, rather than aborting. + +- You can now specify an IPC namespace in `docker-compose.yml` with the `ipc` + option. + +- Containers created by `docker-compose run` can now be named with the + `--name` flag. + +- If you install Compose with pip or use it as a library, it now works with + Python 3. + +- `image` now supports image digests (in addition to ids and tags), e.g. + `image: "busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d"` + +- `ports` now supports ranges of ports, e.g. + + ports: + - "3000-3005" + - "9000-9001:8000-8001" + +- `docker-compose run` now supports a `-p|--publish` parameter, much like + `docker run -p`, for publishing specific ports to the host. + +- `docker-compose pause` and `docker-compose unpause` have been implemented, + analogous to `docker pause` and `docker unpause`. + +- When using `extends` to copy configuration from another service in the same + Compose file, you can omit the `file` option. + +- Compose can be installed and run as a Docker image. This is an experimental feature. +Bug fixes: -Bug Fixes +- All values for the `log_driver` option which are supported by the Docker + daemon are now supported by Compose. -- Support all `log_drivers` -- Fixed `build` when running against swarm -- `~` is no longer expanded on the host when included as part of a container - volume path +- `docker-compose build` can now be run successfully against a Swarm cluster. diff --git a/compose/__init__.py b/compose/__init__.py index 06897c8448..8ea59a363f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0rc1' +__version__ = '1.5.0rc2' diff --git a/docs/install.md b/docs/install.md index 31b2ccad47..3c5dea5eba 100644 --- a/docs/install.md +++ b/docs/install.md @@ -53,7 +53,7 @@ To install Compose, do the following: 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.0rc1 + docker-compose version: 1.5.0rc2 ## Alternative install options @@ -75,7 +75,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc1/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc2/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 68ee4faa5b..25fc8c0770 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0rc1" +VERSION="1.5.0rc2" IMAGE="docker/compose:$VERSION" From 8156cdc56e5b5b1f440b9abbf54faaad14558c23 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 17:54:38 -0400 Subject: [PATCH 307/337] Disable a test against docker 1.8.3 because it fails due to a bug in docker engine. Signed-off-by: Daniel Nephin --- tests/integration/service_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 38d7d5b55b..4ac04545e1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -693,10 +693,11 @@ class ServiceTest(DockerClientTestCase): @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): - """ - Test that calling scale on a service that has a custom container name + """Test that calling scale on a service that has a custom container name results in warning output. """ + # Disable this test against earlier versions because it is flaky + self.require_api_version('1.21') service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name(), 'custom-container') From ce729b07216bd3f6e903829818e946282dbd7e51 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 15:17:11 -0400 Subject: [PATCH 308/337] Update docs about networking for current release. Signed-off-by: Daniel Nephin --- docs/networking.md | 36 ++++++++++++++++++++++-------------- project/ISSUE-TRIAGE.md | 25 +++++++++++++------------ 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/docs/networking.md b/docs/networking.md index 9a6d792df4..718d56c7a2 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -14,7 +14,11 @@ weight=6 > **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. -Compose sets up a single default [network](/engine/reference/commandline/network_create.md) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. +Compose sets up a single default +[network](/engine/reference/commandline/network_create.md) for your app. Each +container for a service joins the default network and is both *reachable* by +other containers on that network, and *discoverable* by them at a hostname +identical to the container name. > **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. @@ -30,13 +34,23 @@ For example, suppose your app is in a directory called `myapp`, and your `docker When you run `docker-compose --x-networking up`, the following happens: 1. A network called `myapp` is created. -2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`. -3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`. +2. A container is created using `web`'s configuration. It joins the network +`myapp` under the name `myapp_web_1`. +3. A container is created using `db`'s configuration. It joins the network +`myapp` under the name `myapp_db_1`. -Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database. +Each container can now look up the hostname `myapp_web_1` or `myapp_db_1` and +get back the appropriate container's IP address. For example, `web`'s +application code could connect to the URL `postgres://myapp_db_1:5432` and start +using the Postgres database. Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. +> **Note:** in the next release there will be additional aliases for the +> container, including a short name without the project name and container +> index. The full container name will remain as one of the alias for backwards +> compatibility. + ## Updating containers If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. @@ -45,19 +59,13 @@ If any containers have connections open to the old container, they will be close ## Configure how services are published -By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option: +By default, containers for each service are published on the network with the +container name. If you want to change the name, or stop containers from being +discoverable at all, you can use the `container_name` option: web: build: . - hostname: "my-web-application" - -This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`. - -## Scaling services - -If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one. - -This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner. + container_name: "my-web-application" ## Links diff --git a/project/ISSUE-TRIAGE.md b/project/ISSUE-TRIAGE.md index 58312a6037..b89cdc240a 100644 --- a/project/ISSUE-TRIAGE.md +++ b/project/ISSUE-TRIAGE.md @@ -20,15 +20,16 @@ The following labels are provided in additional to the standard labels: Most issues should fit into one of the following functional areas: -| Area | -|----------------| -| area/build | -| area/cli | -| area/config | -| area/logs | -| area/packaging | -| area/run | -| area/scale | -| area/tests | -| area/up | -| area/volumes | +| Area | +|-----------------| +| area/build | +| area/cli | +| area/config | +| area/logs | +| area/networking | +| area/packaging | +| area/run | +| area/scale | +| area/tests | +| area/up | +| area/volumes | From f67503d9fd2864fc16cf64811b1b38b6a58ff5e4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 11:28:17 -0400 Subject: [PATCH 309/337] Logs are available for all log drivers except for none. Signed-off-by: Daniel Nephin --- compose/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/container.py b/compose/container.py index dd69e8dddb..1ca483809a 100644 --- a/compose/container.py +++ b/compose/container.py @@ -145,7 +145,7 @@ class Container(object): @property def has_api_logs(self): log_type = self.log_driver - return not log_type or log_type == 'json-file' + return not log_type or log_type != 'none' def attach_log_stream(self): """A log stream can only be attached if the container uses a json-file From ab0ddb593f8a6a98c3bb85b8487574179f8f2262 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 29 Oct 2015 16:52:00 +0000 Subject: [PATCH 310/337] Clarify the command is an example Signed-off-by: Mazz Mosley --- docs/install.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index 3c5dea5eba..ea78948ed7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -30,13 +30,14 @@ To install Compose, do the following: 3. Go to the Compose repository release page on GitHub. -4. Follow the instructions from the release page and run the `curl` command in your terminal. +4. Follow the instructions from the release page and run the `curl` command, +which the release page specifies, in your terminal. > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands below, then `exit`. - The command has the following format: + The following is an example command illustrating the format: curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose From 9370cb033cc7a51e55144085d2e76795c64982af Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Oct 2015 17:56:01 +0100 Subject: [PATCH 311/337] Ensure network exists when calling run before up Otherwise the daemon will error out because the network doesn't exist yet. Signed-off-by: Joffrey F --- compose/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 5505b89f51..4369aa707a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -380,6 +380,8 @@ class TopLevelCommand(DocoptCommand): start_deps=True, strategy=ConvergenceStrategy.never, ) + elif project.use_networking: + project.ensure_network_exists() tty = True if detach or options['-T'] or not sys.stdin.isatty(): From 1f26841e238e4d1f005d194cae6f076b93a3d8a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Oct 2015 14:15:47 +0100 Subject: [PATCH 312/337] Integration test for run command with networking enabled Signed-off-by: Joffrey F --- tests/integration/cli_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 19cc822ee1..45f45645f5 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -508,6 +508,20 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) self.assertEqual(container.name, name) + @mock.patch('dockerpty.start') + def test_run_with_networking(self, _): + self.require_api_version('1.21') + client = docker_client(version='1.21') + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['--x-networking', 'run', 'simple', 'true'], None) + service = self.project.get_service('simple') + container, = service.containers(stopped=True, one_off=True) + networks = client.networks(names=[self.project.name]) + for n in networks: + self.addCleanup(client.remove_network, n['id']) + self.assertEqual(len(networks), 1) + self.assertEqual(container.human_readable_command, u'true') + def test_rm(self): service = self.project.get_service('simple') service.create_container() From 4d613d3ba780f46c46669e829eadad1daf5e0cb4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 14:06:50 -0400 Subject: [PATCH 313/337] Use colors when logging warnings or errors, so they are more obvious. Signed-off-by: Daniel Nephin --- compose/cli/formatter.py | 23 +++++++++++++++++++++++ compose/cli/main.py | 22 ++++++++++++++-------- compose/config/interpolation.py | 2 +- tests/unit/cli/main_test.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index 9ed52c4aa5..d0ed0f87eb 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -1,10 +1,13 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging import os import texttable +from compose.cli import colors + def get_tty_width(): tty_size = os.popen('stty size', 'r').read().split() @@ -15,6 +18,7 @@ def get_tty_width(): class Formatter(object): + """Format tabular data for printing.""" def table(self, headers, rows): table = texttable.Texttable(max_width=get_tty_width()) table.set_cols_dtype(['t' for h in headers]) @@ -23,3 +27,22 @@ class Formatter(object): table.set_chars(['-', '|', '+', '-']) return table.draw() + + +class ConsoleWarningFormatter(logging.Formatter): + """A logging.Formatter which prints WARNING and ERROR messages with + a prefix of the log level colored appropriate for the log level. + """ + + def get_level_message(self, record): + separator = ': ' + if record.levelno == logging.WARNING: + return colors.yellow(record.levelname) + separator + if record.levelno == logging.ERROR: + return colors.red(record.levelname) + separator + + return '' + + def format(self, record): + message = super(ConsoleWarningFormatter, self).format(record) + return self.get_level_message(record) + message diff --git a/compose/cli/main.py b/compose/cli/main.py index 4369aa707a..2a5ab49af8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -28,6 +28,7 @@ from .command import project_from_options from .docopt_command import DocoptCommand from .docopt_command import NoSuchCommand from .errors import UserError +from .formatter import ConsoleWarningFormatter from .formatter import Formatter from .log_printer import LogPrinter from .utils import get_version_info @@ -41,7 +42,7 @@ log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) INSECURE_SSL_WARNING = """ -Warning: --allow-insecure-ssl is deprecated and has no effect. +--allow-insecure-ssl is deprecated and has no effect. It will be removed in a future version of Compose. """ @@ -91,13 +92,18 @@ def setup_logging(): logging.getLogger("requests").propagate = False -def setup_console_handler(verbose): - if verbose: - console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) - console_handler.setLevel(logging.DEBUG) +def setup_console_handler(handler, verbose): + if handler.stream.isatty(): + format_class = ConsoleWarningFormatter else: - console_handler.setFormatter(logging.Formatter()) - console_handler.setLevel(logging.INFO) + format_class = logging.Formatter + + if verbose: + handler.setFormatter(format_class('%(name)s.%(funcName)s: %(message)s')) + handler.setLevel(logging.DEBUG) + else: + handler.setFormatter(format_class()) + handler.setLevel(logging.INFO) # stolen from docopt master @@ -153,7 +159,7 @@ class TopLevelCommand(DocoptCommand): return options def perform_command(self, options, handler, command_options): - setup_console_handler(options.get('--verbose')) + setup_console_handler(console_handler, options.get('--verbose')) if options['COMMAND'] in ('help', 'version'): # Skip looking up the compose file. diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index f870ab4b27..f8e1da610d 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -78,7 +78,7 @@ class BlankDefaultDict(dict): except KeyError: if key not in self.missing_keys: log.warn( - "The {} variable is not set. Substituting a blank string." + "The {} variable is not set. Defaulting to a blank string." .format(key) ) self.missing_keys.append(key) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index a5b369808b..ee837fcd45 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -1,11 +1,15 @@ from __future__ import absolute_import +import logging + from compose import container from compose.cli.errors import UserError +from compose.cli.formatter import ConsoleWarningFormatter from compose.cli.log_printer import LogPrinter from compose.cli.main import attach_to_logs from compose.cli.main import build_log_printer from compose.cli.main import convergence_strategy_from_opts +from compose.cli.main import setup_console_handler from compose.project import Project from compose.service import ConvergenceStrategy from tests import mock @@ -60,6 +64,31 @@ class CLIMainTestCase(unittest.TestCase): timeout=timeout) +class SetupConsoleHandlerTestCase(unittest.TestCase): + + def setUp(self): + self.stream = mock.Mock() + self.stream.isatty.return_value = True + self.handler = logging.StreamHandler(stream=self.stream) + + def test_with_tty_verbose(self): + setup_console_handler(self.handler, True) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' in self.handler.formatter._fmt + assert '%(funcName)s' in self.handler.formatter._fmt + + def test_with_tty_not_verbose(self): + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' not in self.handler.formatter._fmt + assert '%(funcName)s' not in self.handler.formatter._fmt + + def test_with_not_a_tty(self): + self.stream.isatty.return_value = False + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == logging.Formatter + + class ConvergeStrategyFromOptsTestCase(unittest.TestCase): def test_invalid_opts(self): From db164cefd336e152b12e3088aa92d274d852b157 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 14:15:08 -0400 Subject: [PATCH 314/337] Remove the duplicate 'Warning' prefix now that the logger adds the prefix. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 5 ++--- compose/config/validation.py | 8 +++++--- compose/service.py | 2 +- tests/unit/cli/formatter_test.py | 35 ++++++++++++++++++++++++++++++++ tests/unit/config/config_test.py | 2 +- 5 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 tests/unit/cli/formatter_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 2a5ab49af8..b54b307ef2 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -59,9 +59,8 @@ def main(): log.error(e.msg) sys.exit(1) except NoSuchCommand as e: - log.error("No such command: %s", e.command) - log.error("") - log.error("\n".join(parse_doc_section("commands:", getdoc(e.supercommand)))) + commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) + log.error("No such command: %s\n\n%s", e.command, commands) sys.exit(1) except APIError as e: log.error(e.explanation) diff --git a/compose/config/validation.py b/compose/config/validation.py index 8cfc405fe8..542081d526 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -57,9 +57,11 @@ def format_boolean_in_environment(instance): """ if isinstance(instance, bool): log.warn( - "Warning: There is a boolean value in the 'environment' key.\n" - "Environment variables can only be strings.\nPlease add quotes to any boolean values to make them string " - "(eg, 'True', 'yes', 'N').\nThis warning will become an error in a future release. \r\n" + "There is a boolean value in the 'environment' key.\n" + "Environment variables can only be strings.\n" + "Please add quotes to any boolean values to make them string " + "(eg, 'True', 'yes', 'N').\n" + "This warning will become an error in a future release. \r\n" ) return True diff --git a/compose/service.py b/compose/service.py index dbcec10382..66c90b0e03 100644 --- a/compose/service.py +++ b/compose/service.py @@ -848,7 +848,7 @@ class ServiceNet(object): if containers: return 'container:' + containers[0].id - log.warn("Warning: Service %s is trying to use reuse the network stack " + log.warn("Service %s is trying to use reuse the network stack " "of another service that is not running." % (self.id)) return None diff --git a/tests/unit/cli/formatter_test.py b/tests/unit/cli/formatter_test.py new file mode 100644 index 0000000000..1c3b6a68ef --- /dev/null +++ b/tests/unit/cli/formatter_test.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging + +from compose.cli import colors +from compose.cli.formatter import ConsoleWarningFormatter +from tests import unittest + + +MESSAGE = 'this is the message' + + +def makeLogRecord(level): + return logging.LogRecord('name', level, 'pathame', 0, MESSAGE, (), None) + + +class ConsoleWarningFormatterTestCase(unittest.TestCase): + + def setUp(self): + self.formatter = ConsoleWarningFormatter() + + def test_format_warn(self): + output = self.formatter.format(makeLogRecord(logging.WARN)) + expected = colors.yellow('WARNING') + ': ' + assert output == expected + MESSAGE + + def test_format_error(self): + output = self.formatter.format(makeLogRecord(logging.ERROR)) + expected = colors.red('ERROR') + ': ' + assert output == expected + MESSAGE + + def test_format_info(self): + output = self.formatter.format(makeLogRecord(logging.INFO)) + assert output == MESSAGE diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a54b006fa6..2835e9c805 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -380,7 +380,7 @@ class ConfigTest(unittest.TestCase): @mock.patch('compose.config.validation.log') def test_logs_warning_for_boolean_in_environment(self, mock_logging): - expected_warning_msg = "Warning: There is a boolean value in the 'environment' key." + expected_warning_msg = "There is a boolean value in the 'environment' key." config.load( build_config_details( {'web': { From d392f70cc6aa88f60f40cf2e58d9603ebac847ac Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 15:04:35 -0400 Subject: [PATCH 315/337] Fixes #1843, #1936 - chown files back to host user in django example. Also add a missing 'touch Gemfile.lock' to fix the rails tutorial. Signed-off-by: Daniel Nephin --- docs/django.md | 16 ++++++++++++++-- docs/rails.md | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/django.md b/docs/django.md index fd18784ecc..2bb67399c5 100644 --- a/docs/django.md +++ b/docs/django.md @@ -110,8 +110,20 @@ In this step, you create a Django started project by building the image from the 3. After the `docker-compose` command completes, list the contents of your project. - $ ls - Dockerfile docker-compose.yml composeexample manage.py requirements.txt + $ ls -l + drwxr-xr-x 2 root root composeexample + -rw-rw-r-- 1 user user docker-compose.yml + -rw-rw-r-- 1 user user Dockerfile + -rwxr-xr-x 1 root root manage.py + -rw-rw-r-- 1 user user requirements.txt + + The files `django-admin` created are owned by root. This happens because + the container runs as the `root` user. + +4. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + ## Connect the database diff --git a/docs/rails.md b/docs/rails.md index a33cac26ed..e81675c537 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,6 +37,10 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' +You'll need an empty `Gemfile.lock` in order to build our `Dockerfile`. + + $ touch Gemfile.lock + Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. db: @@ -69,6 +73,12 @@ image. Once it's done, you should have generated a fresh app: README.rdoc config.ru public Rakefile db test + +The files `rails new` created are owned by root. This happens because the +container runs as the `root` user. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + Uncomment the line in your new `Gemfile` which loads `therubyracer`, so you've got a Javascript runtime: @@ -80,6 +90,7 @@ rebuild.) $ docker-compose build + ### Connect the database The app is now bootable, but you're not quite there yet. By default, Rails @@ -87,8 +98,7 @@ expects a database to be running on `localhost` - so you need to point it at the `db` container instead. You also need to change the database and username to align with the defaults set by the `postgres` image. -Open up your newly-generated `database.yml` file. Replace its contents with the -following: +Replace the contents of `config/database.yml` with the following: development: &default adapter: postgresql From 2f2e946907e463404e36525e91ec90f05b606e29 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 11:53:36 -0400 Subject: [PATCH 316/337] Don't set a default network driver, let the server decide. Signed-off-by: Daniel Nephin --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 68edaddcb2..1e01eaf6d2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -83,7 +83,7 @@ class Project(object): self.services = services self.client = client self.use_networking = use_networking - self.network_driver = network_driver or 'bridge' + self.network_driver = network_driver def labels(self, one_off=False): return [ From ed1b584c42c5acd6f33278df48939957720319b4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:01:34 -0400 Subject: [PATCH 317/337] Fix release script notes about software and typos. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 12 ++++++++++-- script/release/cherry-pick-pr | 2 +- script/release/push-release | 5 ++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index ffa18077f4..040a2602be 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -1,6 +1,14 @@ Building a Compose release ========================== +## Prerequisites + +The release scripts require the following tools installed on the host: + +* https://hub.github.com/ +* https://stedolan.github.io/jq/ +* http://pandoc.org/ + ## To get started with a new release Create a branch, update version, and add release notes by running `make-branch` @@ -40,10 +48,10 @@ As part of this script you'll be asked to: ## To release a version (whether RC or stable) -Check out the bump branch and run the `build-binary` script +Check out the bump branch and run the `build-binaries` script git checkout bump-$VERSION - ./script/release/build-binary + ./script/release/build-binaries When prompted build the non-linux binaries and test them. diff --git a/script/release/cherry-pick-pr b/script/release/cherry-pick-pr index 604600872c..f4a5a7406b 100755 --- a/script/release/cherry-pick-pr +++ b/script/release/cherry-pick-pr @@ -22,7 +22,7 @@ EOM if [ -z "$(command -v hub 2> /dev/null)" ]; then >&2 echo "$0 requires https://hub.github.com/." - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi diff --git a/script/release/push-release b/script/release/push-release index 039436da0e..9229f0934d 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -34,7 +34,9 @@ GITHUB_REPO=git@github.com:$REPO sha=$(git rev-parse HEAD) url=$API/$REPO/statuses/$sha build_status=$(curl -s $url | jq -r '.[0].state') -if [[ "$build_status" != "success" ]]; then +if [ -n "$SKIP_BUILD_CHECK" ]; then + echo "Skipping build status check..." +elif [[ "$build_status" != "success" ]]; then >&2 echo "Build status is $build_status, but it should be success." exit -1 fi @@ -61,6 +63,7 @@ source venv-test/bin/activate pip install docker-compose==$VERSION docker-compose version deactivate +rm -rf venv-test echo "Now publish the github release, and test the downloads." echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release." From 569ccbadec9edb396135ec991417e340d2bd56ee Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:11:29 -0400 Subject: [PATCH 318/337] Convert the README to rst and fix the logo url before packaging it up for pypi. Signed-off-by: Daniel Nephin --- .gitignore | 1 + MANIFEST.in | 2 ++ script/release/push-release | 15 +++++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1b0c50113f..83a08a0e69 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /dist /docs/_site /venv +README.rst diff --git a/MANIFEST.in b/MANIFEST.in index 43ae06d3e2..0342e35bea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,8 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md +exclude README.md +include README.rst include compose/config/*.json recursive-include contrib/completion * recursive-include tests * diff --git a/script/release/push-release b/script/release/push-release index 9229f0934d..ccdf249607 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -21,11 +21,17 @@ VERSION="$(git config "branch.${BRANCH}.release")" || usage if [ -z "$(command -v jq 2> /dev/null)" ]; then >&2 echo "$0 requires https://stedolan.github.io/jq/" - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi +if [ -z "$(command -v pandoc 2> /dev/null)" ]; then + >&2 echo "$0 requires http://pandoc.org/" + >&2 echo "Please install it and make sure it is available on your \$PATH." + exit 2 +fi + API=https://api.github.com/repos REPO=docker/compose GITHUB_REPO=git@github.com:$REPO @@ -45,12 +51,13 @@ echo "Tagging the release as $VERSION" git tag $VERSION git push $GITHUB_REPO $VERSION -echo "Uploading sdist to pypi" -python setup.py sdist - echo "Uploading the docker image" docker push docker/compose:$VERSION +echo "Uploading sdist to pypi" +pandoc -f markdown -t rst README.md -o README.rst +sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst +python setup.py sdist if [ "$(command -v twine 2> /dev/null)" ]; then twine upload ./dist/docker-compose-${VERSION}.tar.gz else From 73ca4eb5991dda6986d4970635a2f6aea463f8e4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:40:59 -0400 Subject: [PATCH 319/337] On error print daemon logs Signed-off-by: Daniel Nephin --- script/test-versions | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/script/test-versions b/script/test-versions index 89793359be..43326ccb6b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -28,10 +28,15 @@ for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" daemon_container="compose-dind-$version-$BUILD_NUMBER" - trap "docker rm -vf $daemon_container" EXIT - # TODO: remove when we stop testing against 1.7.x - daemon=$([[ "$version" == "1.7"* ]] && echo "-d" || echo "daemon") + function on_exit() { + if [[ "$?" != "0" ]]; then + docker logs "$daemon_container" + fi + docker rm -vf "$daemon_container" + } + + trap "on_exit" EXIT docker run \ -d \ @@ -39,7 +44,7 @@ for version in $DOCKER_VERSIONS; do --privileged \ --volume="/var/lib/docker" \ dockerswarm/dind:$version \ - docker $daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ docker run \ --rm \ From bdb9a280bc8e42ac79dc453c44b4ceb74f1aaee6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 15:26:44 -0400 Subject: [PATCH 320/337] Make storage driver configurable in CI Signed-off-by: Daniel Nephin --- script/ci | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/ci b/script/ci index 12dc3c473e..f30265c02a 100755 --- a/script/ci +++ b/script/ci @@ -11,7 +11,9 @@ set -ex docker version export DOCKER_VERSIONS=all -export DOCKER_DAEMON_ARGS="--storage-driver=overlay" +STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} +export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" + GIT_VOLUME="--volumes-from=$(hostname)" . script/test-versions From be6b811c4e9a1e8f22f2216c128b9bc91f4ebfdd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 12:51:57 -0400 Subject: [PATCH 321/337] Bump 1.5.0rc3 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 20 +++++++++++++++++--- compose/__init__.py | 2 +- docs/install.md | 4 ++-- script/run.sh | 2 +- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b2ecd97e6..b0474ae2b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,33 @@ Change log ========== -1.5.0 (2015-10-13) +1.5.0 (2015-11-02) ------------------ +**Breaking changes:** + +With the introduction of variable substitution support in the Compose file, any +Compose file that uses an environment variable (`$VAR` or `${VAR}`) in the `command:` +or `entrypoint:` field will break. + +Previously these values were interpolated inside the container, with a value +from the container environment. In Compose 1.5.0, the values will be +interpolated on the host, with a value from the host environment. + +To migrate a Compose file to 1.5.0, escape the variables with an extra `$` +(ex: `$$VAR` or `$${VAR}`). See +https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution + Major features: - Compose is now available for Windows. - Environment variables can be used in the Compose file. See - https://github.com/docker/compose/blob/129092b7/docs/yml.md#variable-substitution + https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution - Multiple compose files can be specified, allowing you to override settings in the default Compose file. See - https://github.com/docker/compose/blob/129092b7/docs/reference/docker-compose.md + https://github.com/docker/compose/blob/8cc8e61/docs/reference/docker-compose.md for more details. - Compose now produces better error messages when a file contains diff --git a/compose/__init__.py b/compose/__init__.py index 8ea59a363f..7199babb40 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0rc2' +__version__ = '1.5.0rc3' diff --git a/docs/install.md b/docs/install.md index ea78948ed7..711902c7fa 100644 --- a/docs/install.md +++ b/docs/install.md @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.0rc2 + docker-compose version: 1.5.0rc3 ## Alternative install options @@ -76,7 +76,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc2/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc3/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 25fc8c0770..9ed1ea74cf 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0rc2" +VERSION="1.5.0rc3" IMAGE="docker/compose:$VERSION" From e524cce222440f21740925a6e247b7d122f7c4c6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 16:29:36 -0400 Subject: [PATCH 322/337] Add missing title to compose file reference. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/compose-file.md b/docs/compose-file.md index b72a7cc437..ffcda61cb9 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -24,6 +24,11 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +## Service configuration reference + +This section contains a list of all configuration options supported by a service +definition. + ### build Path to a directory containing a Dockerfile. When the value supplied is a From 8733d09a9c7b3a53fa7eef0d13d58c9dde7a4b30 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 10:49:41 -0400 Subject: [PATCH 323/337] Extract the getting started guide from the index page. Signed-off-by: Daniel Nephin --- docs/completion.md | 2 +- docs/django.md | 1 + docs/extends.md | 1 + docs/gettingstarted.md | 163 +++++++++++++++++++++++++++++++++++++++++ docs/index.md | 139 +---------------------------------- docs/install.md | 1 + docs/production.md | 3 - docs/rails.md | 2 +- docs/wordpress.md | 2 +- 9 files changed, 170 insertions(+), 144 deletions(-) create mode 100644 docs/gettingstarted.md diff --git a/docs/completion.md b/docs/completion.md index bc8bedc96c..3c2022d827 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -5,7 +5,7 @@ description = "Compose CLI reference" keywords = ["fig, composition, compose, docker, orchestration, cli, reference"] [menu.main] parent="smn_workw_compose" -weight=3 +weight=10 +++ diff --git a/docs/django.md b/docs/django.md index 2bb67399c5..d4d2bd1ecf 100644 --- a/docs/django.md +++ b/docs/django.md @@ -173,6 +173,7 @@ In this section, you set up the database connection for Django. - [User guide](../index.md) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) diff --git a/docs/extends.md b/docs/extends.md index f0b9e9ea2d..e63cf4662e 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -360,6 +360,7 @@ locally-defined bindings taking precedence: - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md new file mode 100644 index 0000000000..f2024b39ba --- /dev/null +++ b/docs/gettingstarted.md @@ -0,0 +1,163 @@ + + + +## Getting Started + +Let's get started with a walkthrough of getting a simple Python web app running +on Compose. It assumes a little knowledge of Python, but the concepts +demonstrated here should be understandable even if you're not familiar with +Python. + +### Installation and set-up + +First, [install Docker and Compose](install.md). + +Next, you'll want to make a directory for the project: + + $ mkdir composetest + $ cd composetest + +Inside this directory, create `app.py`, a simple Python web app that uses the Flask +framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): + + from flask import Flask + from redis import Redis + + app = Flask(__name__) + redis = Redis(host='redis', port=6379) + + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.' % redis.get('hits') + + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) + +Next, define the Python dependencies in a file called `requirements.txt`: + + flask + redis + +### Create a Docker image + +Now, create a Docker image containing all of your app's dependencies. You +specify how to build the image using a file called +[`Dockerfile`](http://docs.docker.com/reference/builder/): + + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r requirements.txt + CMD python app.py + +This tells Docker to: + +* Build an image starting with the Python 2.7 image. +* Add the current directory `.` into the path `/code` in the image. +* Set the working directory to `/code`. +* Install the Python dependencies. +* Set the default command for the container to `python app.py` + +For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + +You can build the image by running `docker build -t web .`. + +### Define services + +Next, define a set of services using `docker-compose.yml`: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis + +This template defines two services, `web` and `redis`. The `web` service: + +* Builds from the `Dockerfile` in the current directory. +* Forwards the exposed port 5000 on the container to port 5000 on the host machine. +* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. +* Links the web container to the Redis service. + +The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. + +### Build and run your app with Compose + +Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: + + $ docker-compose up + Pulling image redis... + Building web... + Starting composetest_redis_1... + Starting composetest_web_1... + redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 + web_1 | * Running on http://0.0.0.0:5000/ + web_1 | * Restarting with stat + +If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. + +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. + +You should get a message in your browser saying: + +`Hello World! I have been seen 1 times.` + +Refreshing the page will increment the number. + +If you want to run your services in the background, you can pass the `-d` flag +(for "detached" mode) to `docker-compose up` and use `docker-compose ps` to +see what is currently running: + + $ docker-compose up -d + Starting composetest_redis_1... + Starting composetest_web_1... + $ docker-compose ps + Name Command State Ports + ------------------------------------------------------------------- + composetest_redis_1 /usr/local/bin/run Up + composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp + +The `docker-compose run` command allows you to run one-off commands for your +services. For example, to see what environment variables are available to the +`web` service: + + $ docker-compose run web env + +See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. + +If you started Compose with `docker-compose up -d`, you'll probably want to stop +your services once you've finished with them: + + $ docker-compose stop + +At this point, you have seen the basics of how Compose works. + +- Next, try the quick start guide for [Django](django.md), + [Rails](rails.md), or [WordPress](wordpress.md). +- See the reference guides for complete details on the [commands](./reference/index.md), the + [configuration file](compose-file.md) and [environment variables](env.md). + +## More Compose documentation + +- [User guide](/) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with WordPress](wordpress.md) +- [Command line reference](./reference/index.md) +- [Compose file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index 62c78d6893..19a6c801c2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,150 +50,13 @@ Compose has commands for managing the whole lifecycle of your application: ## Compose documentation - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) -## Quick start - -Let's get started with a walkthrough of getting a simple Python web app running -on Compose. It assumes a little knowledge of Python, but the concepts -demonstrated here should be understandable even if you're not familiar with -Python. - -### Installation and set-up - -First, [install Docker and Compose](install.md). - -Next, you'll want to make a directory for the project: - - $ mkdir composetest - $ cd composetest - -Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): - - from flask import Flask - from redis import Redis - - app = Flask(__name__) - redis = Redis(host='redis', port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - -Next, define the Python dependencies in a file called `requirements.txt`: - - flask - redis - -### Create a Docker image - -Now, create a Docker image containing all of your app's dependencies. You -specify how to build the image using a file called -[`Dockerfile`](http://docs.docker.com/reference/builder/): - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - -This tells Docker to: - -* Build an image starting with the Python 2.7 image. -* Add the current directory `.` into the path `/code` in the image. -* Set the working directory to `/code`. -* Install the Python dependencies. -* Set the default command for the container to `python app.py` - -For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). - -You can build the image by running `docker build -t web .`. - -### Define services - -Next, define a set of services using `docker-compose.yml`: - - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - redis: - image: redis - -This template defines two services, `web` and `redis`. The `web` service: - -* Builds from the `Dockerfile` in the current directory. -* Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. - -The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. - -### Build and run your app with Compose - -Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: - - $ docker-compose up - Pulling image redis... - Building web... - Starting composetest_redis_1... - Starting composetest_web_1... - redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 - web_1 | * Running on http://0.0.0.0:5000/ - web_1 | * Restarting with stat - -If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. - -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. - -You should get a message in your browser saying: - -`Hello World! I have been seen 1 times.` - -Refreshing the page will increment the number. - -If you want to run your services in the background, you can pass the `-d` flag -(for "detached" mode) to `docker-compose up` and use `docker-compose ps` to -see what is currently running: - - $ docker-compose up -d - Starting composetest_redis_1... - Starting composetest_web_1... - $ docker-compose ps - Name Command State Ports - ------------------------------------------------------------------- - composetest_redis_1 /usr/local/bin/run Up - composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp - -The `docker-compose run` command allows you to run one-off commands for your -services. For example, to see what environment variables are available to the -`web` service: - - $ docker-compose run web env - -See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. - -If you started Compose with `docker-compose up -d`, you'll probably want to stop -your services once you've finished with them: - - $ docker-compose stop - -At this point, you have seen the basics of how Compose works. - -- Next, try the quick start guide for [Django](django.md), - [Rails](rails.md), or [WordPress](wordpress.md). -- See the reference guides for complete details on the [commands](./reference/index.md), the - [configuration file](compose-file.md) and [environment variables](env.md). ## Release Notes diff --git a/docs/install.md b/docs/install.md index 711902c7fa..4eb0dc1869 100644 --- a/docs/install.md +++ b/docs/install.md @@ -127,6 +127,7 @@ To uninstall Docker Compose if you installed using `pip`: ## Where to go next - [User guide](/) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) diff --git a/docs/production.md b/docs/production.md index 8793f9277e..0b0e46c3f0 100644 --- a/docs/production.md +++ b/docs/production.md @@ -86,8 +86,5 @@ guide. ## Compose documentation - [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) diff --git a/docs/rails.md b/docs/rails.md index e81675c537..8e16af6423 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -135,8 +135,8 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) -- [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 8c1f5b0acb..373ef4d0d5 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -95,8 +95,8 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) From 09d2bdbb21135ef88b8eb296f2b6fcd4e6dc03f0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 11:08:23 -0400 Subject: [PATCH 324/337] Flush out features and use cases. Signed-off-by: Daniel Nephin --- README.md | 17 ++++++---- docs/index.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d779d607c3..6b783bf126 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,15 @@ Docker Compose *(Previously known as Fig)* -Compose is a tool for defining and running multi-container applications with -Docker. With Compose, you define a multi-container application in a single -file, then spin your application up in a single command which does everything -that needs to be done to get it running. +Compose is a tool for defining and running multi-container Docker applications. +With Compose, you define a multi-container application in a compose +file then, using a single command, you create and start all the containers +from your configuration. To learn more about all the features of Compose +see [the list of features](#features) -Compose is great for development environments, staging servers, and CI. We don't -recommend that you use it in production yet. +Compose is great for development, testing, and staging environments, as well as +CI workflows. You can learn more about each case in +[Common Use Cases](#common-use-cases). Using Compose is basically a three-step process. @@ -33,6 +35,9 @@ A `docker-compose.yml` looks like this: redis: image: redis +For more information about the Compose file, see the +[Compose file reference](docs/yml.md) + Compose has commands for managing the whole lifecycle of your application: * Start, stop and rebuild services diff --git a/docs/index.md b/docs/index.md index 19a6c801c2..ac7e07f9ba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,20 +11,22 @@ parent="smn_workw_compose" # Overview of Docker Compose -Compose is a tool for defining and running multi-container applications with -Docker. With Compose, you define a multi-container application in a single -file, then spin your application up in a single command which does everything -that needs to be done to get it running. +Compose is a tool for defining and running multi-container Docker applications. +With Compose, you define a multi-container application in a compose +file then, using a single command, you create and start all the containers +from your configuration. To learn more about all the features of Compose +see [the list of features](#features) -Compose is great for development environments, staging servers, and CI. We don't -recommend that you use it in production yet. +Compose is great for development, testing, and staging environments, as well as +CI workflows. You can learn more about each case in +[Common Use Cases](#common-use-cases). Using Compose is basically a three-step process. 1. Define your app's environment with a `Dockerfile` so it can be reproduced anywhere. 2. Define the services that make up your app in `docker-compose.yml` so -they can be run together in an isolated environment: +they can be run together in an isolated environment. 3. Lastly, run `docker-compose up` and Compose will start and run your entire app. A `docker-compose.yml` looks like this: @@ -40,6 +42,9 @@ A `docker-compose.yml` looks like this: redis: image: redis +For more information about the Compose file, see the +[Compose file reference](yml.md) + Compose has commands for managing the whole lifecycle of your application: * Start, stop and rebuild services @@ -57,11 +62,84 @@ Compose has commands for managing the whole lifecycle of your application: - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) +## Features + +#### Preserve volume data + +Compose preserves all volumes used by your services. When `docker-compose up` +runs, if it finds any containers from previous runs, it copies the volumes from +the old container to the new container. This process ensures that any data +you've created in volumes isn't lost. + + +#### Only recreate containers that have changed + +Compose caches the configuration used to create a container. When you +restart a service that has not changed, Compose re-uses the existing +containers. Re-using containers means that you can make changes to your +environment very quickly. + + +#### Variables and moving a composition to different environments + +> New in `docker-compose` 1.5 + +Compose supports variables in the Compose file. You can use these variables +to customize your composition for different environments, or different users. +See [Variable substitution](compose-file.md#variable-substitution) for more +details. + +Compose files can also be extended from other files using the `extends` +field in a compose file, or by using multiple files. See [extends](extends.md) +for more details. + + +## Common Use Cases + +Compose can be used in many different ways. Some common use cases are outlined +below. + +### Development environments + +When you're developing software it is often helpful to be able to run the +application and interact with it. If the application has any service dependencies +(databases, queues, caches, web services, etc) you need a way to document the +dependencies, configuration and operation of each. Compose provides a convenient +format for definition these dependencies (the [Compose file](yml.md)) and a CLI +tool for starting an isolated environment. Compose can replace a multi-page +"developer getting started guide" with a single machine readable configuration +file and a single command `docker-compose up`. + +### Automated testing environments + +An important part of any Continuous Deployment or Continuous Integration process +is the automated test suite. Automated end-to-end testing requires an +environment in which to run tests. Compose provides a convenient way to create +and destroy isolated testing environments for your test suite. By defining the full +environment in a [Compose file](yml.md) you can create and destroy these +environments in just a few commands: + + $ docker-compose up -d + $ ./run_tests + $ docker-compose stop + $ docker-compose rm -f + +### Single host deployments + +Compose has traditionally been focused on development and testing workflows, +but with each release we're making progress on more production-oriented features. +Compose can be used to deploy to a remote docker engine, for example a cloud +instance provisioned with [Docker Machine](https://docs.docker.com/machine/) or +a [Docker Swarm](https://docs.docker.com/swarm/) cluster. + +See [compose in production](production.md) for more details. + ## Release Notes To see a detailed list of changes for past and current releases of Docker -Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). +Compose, please refer to the +[CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). ## Getting help From bfb46b37d318eb7f07c031e59840e50bec1e14a2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 16:28:47 -0400 Subject: [PATCH 325/337] Updates to gettingstarted guide from PR feedback. Signed-off-by: Daniel Nephin --- docs/gettingstarted.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index f2024b39ba..9cc478d7e5 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -21,13 +21,13 @@ Python. First, [install Docker and Compose](install.md). -Next, you'll want to make a directory for the project: +Create a directory for the project: $ mkdir composetest $ cd composetest Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): +framework and increments a value in Redis. from flask import Flask from redis import Redis @@ -74,7 +74,7 @@ You can build the image by running `docker build -t web .`. ### Define services -Next, define a set of services using `docker-compose.yml`: +Define a set of services using `docker-compose.yml`: web: build: . @@ -91,8 +91,8 @@ This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. -* Links the web container to the Redis service. +* Mounts the project directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. +* Links the web service to the Redis service. The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. @@ -113,7 +113,7 @@ If you're using [Docker Machine](https://docs.docker.com/machine), then `docker- If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. -You should get a message in your browser saying: +You will see a message in your browser saying: `Hello World! I have been seen 1 times.` From 7ee36829ac87f6e02d14b09c00a63498832d12d3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 23 Oct 2015 16:51:03 -0400 Subject: [PATCH 326/337] Update intro docs based on feedback. Signed-off-by: Daniel Nephin --- README.md | 4 +- docs/gettingstarted.md | 207 +++++++++++++++++++++++------------------ docs/index.md | 50 ++++++---- 3 files changed, 147 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 6b783bf126..f8a5050e7a 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ Docker Compose ============== ![Docker Compose](logo.png?raw=true "Docker Compose Logo") -*(Previously known as Fig)* - Compose is a tool for defining and running multi-container Docker applications. With Compose, you define a multi-container application in a compose file then, using a single command, you create and start all the containers @@ -36,7 +34,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](docs/yml.md) +[Compose file reference](docs/compose-file.md) Compose has commands for managing the whole lifecycle of your application: diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 9cc478d7e5..f685bf3820 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -10,84 +10,103 @@ weight=3 -## Getting Started +# Getting Started -Let's get started with a walkthrough of getting a simple Python web app running -on Compose. It assumes a little knowledge of Python, but the concepts -demonstrated here should be understandable even if you're not familiar with -Python. +On this page you build a simple Python web application running on Compose. The +application uses the Flask framework and increments a value in Redis. While the +sample uses Python, the concepts demonstrated here should be understandable even +if you're not familiar with it. -### Installation and set-up +## Prerequisites -First, [install Docker and Compose](install.md). +Make sure you have already +[installed both Docker Engine and Docker Compose](install.md). You +don't need to install Python, it is provided by a Docker image. -Create a directory for the project: +## Step 1: Setup - $ mkdir composetest - $ cd composetest +1. Create a directory for the project: -Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. + $ mkdir composetest + $ cd composetest - from flask import Flask - from redis import Redis +2. With your favorite text editor create a file called `app.py` in your project + directory. - app = Flask(__name__) - redis = Redis(host='redis', port=6379) + from flask import Flask + from redis import Redis - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') + app = Flask(__name__) + redis = Redis(host='redis', port=6379) - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.' % redis.get('hits') -Next, define the Python dependencies in a file called `requirements.txt`: + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) - flask - redis +3. Create another file called `requirements.txt` in your project directory and + add the following: -### Create a Docker image + flask + redis -Now, create a Docker image containing all of your app's dependencies. You -specify how to build the image using a file called -[`Dockerfile`](http://docs.docker.com/reference/builder/): + These define the applications dependencies. - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py +## Step 2: Create a Docker image -This tells Docker to: +In this step, you build a new Docker image. The image contains all the +dependencies the Python application requires, including Python itself. -* Build an image starting with the Python 2.7 image. -* Add the current directory `.` into the path `/code` in the image. -* Set the working directory to `/code`. -* Install the Python dependencies. -* Set the default command for the container to `python app.py` +1. In your project directory create a file named `Dockerfile` and add the + following: -For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r requirements.txt + CMD python app.py -You can build the image by running `docker build -t web .`. + This tells Docker to: -### Define services + * Build an image starting with the Python 2.7 image. + * Add the current directory `.` into the path `/code` in the image. + * Set the working directory to `/code`. + * Install the Python dependencies. + * Set the default command for the container to `python app.py` + + For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + +2. Build the image. + + $ docker build -t web . + + This command builds an image named `web` from the contents of the current + directory. The command automatically locates the `Dockerfile`, `app.py`, and + `requirements.txt` files. + + +## Step 3: Define services Define a set of services using `docker-compose.yml`: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis +1. Create a file called docker-compose.yml in your project directory and add + the following: -This template defines two services, `web` and `redis`. The `web` service: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis + +This Compose file defines two services, `web` and `redis`. The web service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. @@ -96,68 +115,74 @@ This template defines two services, `web` and `redis`. The `web` service: The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. -### Build and run your app with Compose +## Step 4: Build and run your app with Compose -Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: +1. From your project directory, start up your application. - $ docker-compose up - Pulling image redis... - Building web... - Starting composetest_redis_1... - Starting composetest_web_1... - redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 - web_1 | * Running on http://0.0.0.0:5000/ - web_1 | * Restarting with stat + $ docker-compose up + Pulling image redis... + Building web... + Starting composetest_redis_1... + Starting composetest_web_1... + redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 + web_1 | * Running on http://0.0.0.0:5000/ + web_1 | * Restarting with stat -If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. + Compose pulls a Redis image, builds an image for your code, and start the + services you defined. -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. +2. Enter `http://0.0.0.0:5000/` in a browser to see the application running. -You will see a message in your browser saying: + If you're using Docker on Linux natively, then the web app should now be + listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 + doesn't resolve, you can also try http://localhost:5000. -`Hello World! I have been seen 1 times.` + If you're using Docker Machine on a Mac, use `docker-machine ip MACHINE_VM` to get + the IP address of your Docker host. Then, `open http://MACHINE_VM_IP:5000` in a + browser. -Refreshing the page will increment the number. + You should see a message in your browser saying: + + `Hello World! I have been seen 1 times.` + +3. Refresh the page. + + The number should increment. + +## Step 5: Experiment with some other commands If you want to run your services in the background, you can pass the `-d` flag (for "detached" mode) to `docker-compose up` and use `docker-compose ps` to see what is currently running: - $ docker-compose up -d - Starting composetest_redis_1... - Starting composetest_web_1... - $ docker-compose ps - Name Command State Ports - ------------------------------------------------------------------- - composetest_redis_1 /usr/local/bin/run Up - composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp + $ docker-compose up -d + Starting composetest_redis_1... + Starting composetest_web_1... + $ docker-compose ps + Name Command State Ports + ------------------------------------------------------------------- + composetest_redis_1 /usr/local/bin/run Up + composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp The `docker-compose run` command allows you to run one-off commands for your services. For example, to see what environment variables are available to the `web` service: - $ docker-compose run web env + $ docker-compose run web env See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. If you started Compose with `docker-compose up -d`, you'll probably want to stop your services once you've finished with them: - $ docker-compose stop + $ docker-compose stop At this point, you have seen the basics of how Compose works. + +## Where to go next + - Next, try the quick start guide for [Django](django.md), [Rails](rails.md), or [WordPress](wordpress.md). -- See the reference guides for complete details on the [commands](./reference/index.md), the - [configuration file](compose-file.md) and [environment variables](env.md). - -## More Compose documentation - -- [User guide](/) -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) +- [Explore the full list of Compose commands](./reference/index.md) +- [Compose configuration file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index ac7e07f9ba..6ea0e99ab5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,7 +43,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](yml.md) +[Compose file reference](compose-file.md) Compose has commands for managing the whole lifecycle of your application: @@ -64,6 +64,12 @@ Compose has commands for managing the whole lifecycle of your application: ## Features +The features of Compose that make it effective are: + +* [Preserve volume data](#preserve-volume-data) +* [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) +* [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) + #### Preserve volume data Compose preserves all volumes used by your services. When `docker-compose up` @@ -80,18 +86,15 @@ containers. Re-using containers means that you can make changes to your environment very quickly. -#### Variables and moving a composition to different environments - -> New in `docker-compose` 1.5 +#### Variables and moving a composition between environments Compose supports variables in the Compose file. You can use these variables to customize your composition for different environments, or different users. See [Variable substitution](compose-file.md#variable-substitution) for more details. -Compose files can also be extended from other files using the `extends` -field in a compose file, or by using multiple files. See [extends](extends.md) -for more details. +You can extend a Compose file using the `extends` field or by creating multiple +Compose files. See [extends](extends.md) for more details. ## Common Use Cases @@ -101,14 +104,19 @@ below. ### Development environments -When you're developing software it is often helpful to be able to run the -application and interact with it. If the application has any service dependencies -(databases, queues, caches, web services, etc) you need a way to document the -dependencies, configuration and operation of each. Compose provides a convenient -format for definition these dependencies (the [Compose file](yml.md)) and a CLI -tool for starting an isolated environment. Compose can replace a multi-page -"developer getting started guide" with a single machine readable configuration -file and a single command `docker-compose up`. +When you're developing software, the ability to run an application in an +isolated environment and interact with it is crucial. The Compose command +line tool can be used to create the environment and interact with it. + +The [Compose file](compose-file.md) provides a way to document and configure +all of the application's service dependencies (databases, queues, caches, +web service APIs, etc). Using the Compose command line tool you can create +and start one or more containers for each dependency with a single command +(`docker-compose up`). + +Together, these features provide a convenient way for developers to get +started on a project. Compose can reduce a multi-page "developer getting +started guide" to a single machine readable Compose file and a few commands. ### Automated testing environments @@ -116,7 +124,7 @@ An important part of any Continuous Deployment or Continuous Integration process is the automated test suite. Automated end-to-end testing requires an environment in which to run tests. Compose provides a convenient way to create and destroy isolated testing environments for your test suite. By defining the full -environment in a [Compose file](yml.md) you can create and destroy these +environment in a [Compose file](compose-file.md) you can create and destroy these environments in just a few commands: $ docker-compose up -d @@ -128,11 +136,13 @@ environments in just a few commands: Compose has traditionally been focused on development and testing workflows, but with each release we're making progress on more production-oriented features. -Compose can be used to deploy to a remote docker engine, for example a cloud -instance provisioned with [Docker Machine](https://docs.docker.com/machine/) or -a [Docker Swarm](https://docs.docker.com/swarm/) cluster. +You can use Compose to deploy to a remote Docker Engine. The Docker Engine may +be a single instance provisioned with +[Docker Machine](https://docs.docker.com/machine/) or an entire +[Docker Swarm](https://docs.docker.com/swarm/) cluster. -See [compose in production](production.md) for more details. +For details on using production-oriented features, see +[compose in production](production.md) in this documentation. ## Release Notes From 413921a287a55e62e9b6ed4f25f419fb7a7b7b1c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 16:51:49 -0400 Subject: [PATCH 327/337] Add another feature to the docs - multiple environments per host. Signed-off-by: Daniel Nephin --- docs/index.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6ea0e99ab5..ebc1320eae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -66,11 +66,29 @@ Compose has commands for managing the whole lifecycle of your application: The features of Compose that make it effective are: -* [Preserve volume data](#preserve-volume-data) +* [Multiple isolated environments on a single host](#Multiple-isolated-environments-on-a-single-host) +* [Preserve volume data when containers are created](#preserve-volume-data-when-containers-are-created) * [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) * [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) -#### Preserve volume data +#### Multiple isolated environments on a single host + +Compose uses a project name to isolate environments from each other. You can use +this project name to: + +* on a dev host, to create multiple copies of a single environment (ex: you want + to run a stable copy for each feature branch of a project) +* on a CI server, to keep builds from interfering with each other, you can set + the project name to a unique build number +* on a shared host or dev host, to prevent different projects which may use the + same service names, from interfering with each other + +The default project name is the basename of the project directory. You can set +a custom project name by using the +[`-p` command line option](./reference/docker-compose.md) or the +[`COMPOSE_PROJECT_NAME` environment variable](./reference/overview.md#compose-project-name). + +#### Preserve volume data when containers are created Compose preserves all volumes used by your services. When `docker-compose up` runs, if it finds any containers from previous runs, it copies the volumes from From 83714fbac26cad2c1fa0b0add01eb04042f29758 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 13:27:06 -0400 Subject: [PATCH 328/337] Touch up intro paragraph with feedback from @moxiegirl. Signed-off-by: Daniel Nephin --- README.md | 6 +++--- docs/index.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f8a5050e7a..4c967aebcc 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ Docker Compose ![Docker Compose](logo.png?raw=true "Docker Compose Logo") Compose is a tool for defining and running multi-container Docker applications. -With Compose, you define a multi-container application in a compose -file then, using a single command, you create and start all the containers +With Compose, you use a Compose file to configure your application's services. +Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](#features) +see [the list of features](docs/index.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in diff --git a/docs/index.md b/docs/index.md index ebc1320eae..279154eef9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,10 +12,10 @@ parent="smn_workw_compose" # Overview of Docker Compose Compose is a tool for defining and running multi-container Docker applications. -With Compose, you define a multi-container application in a compose -file then, using a single command, you create and start all the containers +With Compose, you use a Compose file to configure your application's services. +Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](#features) +see [the list of features](#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in From 621d1a51679e42a4fed2a1462332e03e6577d3f3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 15:26:56 -0400 Subject: [PATCH 329/337] Fix networking tests to work with new API in engine rc4 (https://github.com/docker/docker/pull/17536) Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 12 ++++++------ tests/integration/project_test.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 45f45645f5..d621f2d132 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -215,17 +215,17 @@ class CLITestCase(DockerClientTestCase): networks = client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['id']) + self.addCleanup(client.remove_network, n['Id']) self.assertEqual(len(networks), 1) - self.assertEqual(networks[0]['driver'], 'bridge') + self.assertEqual(networks[0]['Driver'], 'bridge') - network = client.inspect_network(networks[0]['id']) - self.assertEqual(len(network['containers']), len(services)) + network = client.inspect_network(networks[0]['Id']) + self.assertEqual(len(network['Containers']), len(services)) for service in services: containers = service.containers() self.assertEqual(len(containers), 1) - self.assertIn(containers[0].id, network['containers']) + self.assertIn(containers[0].id, network['Containers']) self.assertEqual(containers[0].get('Config.Hostname'), service.name) web_container = self.project.get_service('web').containers()[0] @@ -518,7 +518,7 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) networks = client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['id']) + self.addCleanup(client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(container.human_readable_command, u'true') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index fd45b9393f..950523878e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -111,7 +111,7 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_does_exist' project = Project(network_name, [], client) client.create_network(network_name) - assert project.get_network()['name'] == network_name + assert project.get_network()['Name'] == network_name def test_net_from_service(self): project = Project.from_dicts( From 9286e62449da0322336728fc2035201939f6a903 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 13:43:29 -0500 Subject: [PATCH 330/337] On a test failure only show the last 100 lines of daemon output. Signed-off-by: Daniel Nephin --- script/build-linux-inner | 2 +- script/test-versions | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/build-linux-inner b/script/build-linux-inner index 1d0f790504..01137ff240 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -8,7 +8,7 @@ VENV=/code/.tox/py27 mkdir -p `pwd`/dist chmod 777 `pwd`/dist -$VENV/bin/pip install -r requirements-build.txt +$VENV/bin/pip install -q -r requirements-build.txt su -c "$VENV/bin/pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version diff --git a/script/test-versions b/script/test-versions index 43326ccb6b..623b107b93 100755 --- a/script/test-versions +++ b/script/test-versions @@ -31,7 +31,7 @@ for version in $DOCKER_VERSIONS; do function on_exit() { if [[ "$?" != "0" ]]; then - docker logs "$daemon_container" + docker logs "$daemon_container" 2>&1 | tail -n 100 fi docker rm -vf "$daemon_container" } @@ -45,6 +45,7 @@ for version in $DOCKER_VERSIONS; do --volume="/var/lib/docker" \ dockerswarm/dind:$version \ docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + 2>&1 | tail -n 10 docker run \ --rm \ From bd3589689221af154f9b321c41f2d69b89aeb268 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 12:34:55 -0400 Subject: [PATCH 331/337] Remove duplication from extends docs. Start restructuring extends docs in preparation for adding documentation about using multiple compose files. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 51 ++++----- docs/extends.md | 240 +++++++++++-------------------------------- 2 files changed, 80 insertions(+), 211 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index ffcda61cb9..33e7d2b53c 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -166,44 +166,29 @@ accessible to linked services. Only the internal port can be specified. Extend another service, in the current file or another, optionally overriding configuration. -Here's a simple example. Suppose we have 2 files - **common.yml** and -**development.yml**. We can use `extends` to define a service in -**development.yml** which uses configuration defined in **common.yml**: +You can use `extends` on any service together with other configuration keys. +The value must be a dictionary with the key: `service` and may optionally have +the `file` key. -**common.yml** + extends: + file: common.yml + service: webapp - webapp: - build: ./webapp - environment: - - DEBUG=false - - SEND_EMAILS=false +The `file` key specifies the location of a Compose configuration file defining +the service which is being extended. The `file` value can be an absolute or +relative path. If you specify a relative path, Docker Compose treats it as +relative to the location of the current file. If you don't specify a `file`, +Compose looks in the current configuration file. -**development.yml** +The `service` key specifies the name of the service to extend, for example `web` +or `database`. - web: - extends: - file: common.yml - service: webapp - ports: - - "8000:8000" - links: - - db - environment: - - DEBUG=true - db: - image: postgres +You can extend a service that itself extends another. You can extend +indefinitely. Compose does not support circular references and `docker-compose` +returns an error if it encounters one. -Here, the `web` service in **development.yml** inherits the configuration of -the `webapp` service in **common.yml** - the `build` and `environment` keys - -and adds `ports` and `links` configuration. It overrides one of the defined -environment variables (DEBUG) with a new value, and the other one -(SEND_EMAILS) is left untouched. - -The `file` key is optional, if it is not set then Compose will look for the -service within the current file. - -For more on `extends`, see the [tutorial](extends.md#example) and -[reference](extends.md#reference). +For more on `extends`, see the +[the extends documentation](extends.md#extending-services). ### external_links diff --git a/docs/extends.md b/docs/extends.md index e63cf4662e..c97b2b4fab 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -10,20 +10,29 @@ weight=2 -## Extending services in Compose +## Extending services and Compose files + +Compose supports two ways to sharing common configuration and +extend a service with that shared configuration. + +1. Extending individual services with [the `extends` field](#extending-services) +2. Extending entire compositions by + [exnteding compose files](#extending-compose-files) + +### Extending services Docker Compose's `extends` keyword enables sharing of common configurations among different files, or even different projects entirely. Extending services -is useful if you have several applications that reuse commonly-defined services. -Using `extends` you can define a service in one place and refer to it from -anywhere. +is useful if you have several services that reuse a common set of configuration +options. Using `extends` you can define a common set of service options in one +place and refer to it from anywhere. -Alternatively, you can deploy the same application to multiple environments with -a slightly different set of services in each case (or with changes to the -configuration of some services). Moreover, you can do so without copy-pasting -the configuration around. +> **Note:** `links` and `volumes_from` are never shared between services using +> `extends`. See +> [Adding and overriding configuration](#adding-and-overriding-configuration) +> for more information. -### Understand the extends configuration +#### Understand the extends configuration When defining any service in `docker-compose.yml`, you can declare that you are extending another service like this: @@ -77,183 +86,46 @@ You can also write other services and link your `web` service to them: db: image: postgres -For full details on how to use `extends`, refer to the [reference](#reference). +#### Example use case -### Example use case +Extending an individual service is useful when you have multiple services that +have a common configuration. In this example we have a composition that with +a web application and a queue worker. Both services use the same codebase and +share many configuration options. -In this example, you’ll repurpose the example app from the [quick start -guide](/). (If you're not familiar with Compose, it's recommended that -you go through the quick start first.) This example assumes you want to use -Compose both to develop an application locally and then deploy it to a -production environment. +In a **common.yml** we'll define the common configuration: -The local and production environments are similar, but there are some -differences. In development, you mount the application code as a volume so that -it can pick up changes; in production, the code should be immutable from the -outside. This ensures it’s not accidentally changed. The development environment -uses a local Redis container, but in production another team manages the Redis -service, which is listening at `redis-production.example.com`. + app: + build: . + environment: + CONFIG_FILE_PATH: /code/config + API_KEY: xxxyyy + cpu_shares: 5 -To configure with `extends` for this sample, you must: - -1. Define the web application as a Docker image in `Dockerfile` and a Compose - service in `common.yml`. - -2. Define the development environment in the standard Compose file, - `docker-compose.yml`. - - - Use `extends` to pull in the web service. - - Configure a volume to enable code reloading. - - Create an additional Redis service for the application to use locally. - -3. Define the production environment in a third Compose file, `production.yml`. - - - Use `extends` to pull in the web service. - - Configure the web service to talk to the external, production Redis service. - -#### Define the web app - -Defining the web application requires the following: - -1. Create an `app.py` file. - - This file contains a simple Python application that uses Flask to serve HTTP - and increments a counter in Redis: - - from flask import Flask - from redis import Redis - import os - - app = Flask(__name__) - redis = Redis(host=os.environ['REDIS_HOST'], port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.\n' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - - This code uses a `REDIS_HOST` environment variable to determine where to - find Redis. - -2. Define the Python dependencies in a `requirements.txt` file: - - flask - redis - -3. Create a `Dockerfile` to build an image containing the app: - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - -4. Create a Compose configuration file called `common.yml`: - - This configuration defines how to run the app. - - web: - build: . - ports: - - "5000:5000" - - Typically, you would have dropped this configuration into - `docker-compose.yml` file, but in order to pull it into multiple files with - `extends`, it needs to be in a separate file. - -#### Define the development environment - -1. Create a `docker-compose.yml` file. - - The `extends` option pulls in the `web` service from the `common.yml` file - you created in the previous section. - - web: - extends: - file: common.yml - service: web - volumes: - - .:/code - links: - - redis - environment: - - REDIS_HOST=redis - redis: - image: redis - - The new addition defines a `web` service that: - - - Fetches the base configuration for `web` out of `common.yml`. - - Adds `volumes` and `links` configuration to the base (`common.yml`) - configuration. - - Sets the `REDIS_HOST` environment variable to point to the linked redis - container. This environment uses a stock `redis` image from the Docker Hub. - -2. Run `docker-compose up`. - - Compose creates, links, and starts a web and redis container linked together. - It mounts your application code inside the web container. - -3. Verify that the code is mounted by changing the message in - `app.py`—say, from `Hello world!` to `Hello from Compose!`. - - Don't forget to refresh your browser to see the change! - -#### Define the production environment - -You are almost done. Now, define your production environment: - -1. Create a `production.yml` file. - - As with `docker-compose.yml`, the `extends` option pulls in the `web` service - from `common.yml`. - - web: - extends: - file: common.yml - service: web - environment: - - REDIS_HOST=redis-production.example.com - -2. Run `docker-compose -f production.yml up`. - - Compose creates *just* a web container and configures the Redis connection via - the `REDIS_HOST` environment variable. This variable points to the production - Redis instance. - - > **Note**: If you try to load up the webapp in your browser you'll get an - > error—`redis-production.example.com` isn't actually a Redis server. - -You've now done a basic `extends` configuration. As your application develops, -you can make any necessary changes to the web service in `common.yml`. Compose -picks up both the development and production environments when you next run -`docker-compose`. You don't have to do any copy-and-paste, and you don't have to -manually keep both environments in sync. +In a **docker-compose.yml** we'll define the concrete services which use the +common configuration: -### Reference + webapp: + extends: + file: common.yml + service: app + command: /code/run_web_app + ports: + - 8080:8080 + links: + - queue + - db -You can use `extends` on any service together with other configuration keys. It -expects a dictionary that contains a `service` key and optionally a `file` key. -The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. + queue_worker: + extends: + file: common.yml + service: app + command: /code/run_worker + links: + - queue -The `file` key specifies the location of a Compose configuration file defining -the extension. The `file` value can be an absolute or relative path. If you -specify a relative path, Docker Compose treats it as relative to the location -of the current file. If you don't specify a `file`, Compose looks in the -current configuration file. - -The `service` key specifies the name of the service to extend, for example `web` -or `database`. - -You can extend a service that itself extends another. You can extend -indefinitely. Compose does not support circular references and `docker-compose` -returns an error if it encounters them. - -#### Adding and overriding configuration +#### Adding and overriding configuration Compose copies configurations from the original service over to the local one, **except** for `links` and `volumes_from`. These exceptions exist to avoid @@ -282,6 +154,8 @@ listed below.** In the case of `build` and `image`, using one in the local service causes Compose to discard the other, if it was defined in the original service. +Example of image replacing build: + # original service build: . @@ -291,6 +165,9 @@ Compose to discard the other, if it was defined in the original service. # result image: redis + +Example of build replacing image: + # original service image: redis @@ -356,6 +233,13 @@ locally-defined bindings taking precedence: - /local-dir/bar:/bar - /local-dir/baz/:baz + +### Extending Compose files + +> **Note:** This feature is new in `docker-compose` 1.5 + + + ## Compose documentation - [User guide](/) From 887c6753f800f1b11a49f17fe42559cf95c6ae61 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 15:21:06 -0400 Subject: [PATCH 332/337] Support a volume to the docs directory and add --watch, so docs can be refreshed. Signed-off-by: Daniel Nephin --- docs/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 021e8f6e5e..b9ef054828 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,8 +13,8 @@ DOCKER_ENVS := \ -e TIMEOUT # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds -# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs) -DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR)) +# to allow `make DOCSDIR=1 docs-shell` (to create a bind mount in docs) +DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR):/docs/content/compose) # to allow `make DOCSPORT=9000 docs` DOCSPORT := 8000 @@ -37,7 +37,7 @@ GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) default: docs docs: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) --watch docs-draft: docs-build $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) From dbd6c62b70c451897178f6e56392947acafebea3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 17:17:38 -0400 Subject: [PATCH 333/337] Changes to production.md for working with multiple Compose files. Signed-off-by: Daniel Nephin --- docs/production.md | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/docs/production.md b/docs/production.md index 0b0e46c3f0..39f0e1fe19 100644 --- a/docs/production.md +++ b/docs/production.md @@ -12,11 +12,9 @@ weight=1 ## Using Compose in production -While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide -can help. -The project is actively working towards becoming -production-ready; to learn more about the progress being made, check out the roadmap for details -on how it's coming along and what still needs to be done. +> Compose is still primarily aimed at development and testing environments. +> Compose may be used for smaller production deployments, but is probably +> not yet suitable for larger deployments. When deploying to production, you'll almost certainly want to make changes to your app configuration that are more appropriate to a live environment. These @@ -30,22 +28,16 @@ changes may include: - Specifying a restart policy (e.g., `restart: always`) to avoid downtime - Adding extra services (e.g., a log aggregator) -For this reason, you'll probably want to define a separate Compose file, say -`production.yml`, which specifies production-appropriate configuration. +For this reason, you'll probably want to define an additional Compose file, say +`production.yml`, which specifies production-appropriate +configuration. This configuration file only needs to include the changes you'd +like to make from the original Compose file. The additional Compose file +can be applied over the original `docker-compose.yml` to create a new configuration. -> **Note:** The [extends](extends.md) keyword is useful for maintaining multiple -> Compose files which re-use common services without having to manually copy and -> paste. +Once you've got a second configuration file, tell Compose to use it with the +`-f` option: -Once you've got an alternate configuration file, make Compose use it -by setting the `COMPOSE_FILE` environment variable: - - $ export COMPOSE_FILE=production.yml - $ docker-compose up -d - -> **Note:** You can also use the file for a one-off command without setting -> an environment variable. You do this by passing the `-f` flag, e.g., -> `docker-compose -f production.yml up -d`. + $ docker-compose -f docker-compose.yml -f production.yml up -d ### Deploying changes From 58de4e0c267ebf34d053bf30e68acd1600544b68 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 12:56:59 -0400 Subject: [PATCH 334/337] Document using multiple Compose files use cases. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 18 ++-- docs/extends.md | 208 ++++++++++++++++++++++++++++++++++--------- docs/production.md | 3 + 3 files changed, 176 insertions(+), 53 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 33e7d2b53c..034653efe8 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -167,21 +167,21 @@ Extend another service, in the current file or another, optionally overriding configuration. You can use `extends` on any service together with other configuration keys. -The value must be a dictionary with the key: `service` and may optionally have -the `file` key. +The `extends` value must be a dictionary defined with a required `service` +and an optional `file` key. extends: file: common.yml service: webapp -The `file` key specifies the location of a Compose configuration file defining -the service which is being extended. The `file` value can be an absolute or -relative path. If you specify a relative path, Docker Compose treats it as -relative to the location of the current file. If you don't specify a `file`, -Compose looks in the current configuration file. +The `service` the name of the service being extended, for example +`web` or `database`. The `file` is the location of a Compose configuration +file defining that service. -The `service` key specifies the name of the service to extend, for example `web` -or `database`. +If you omit the `file` Compose looks for the service configuration in the +current file. The `file` value can be an absolute or relative path. If you +specify a relative path, Compose treats it as relative to the location of the +current file. You can extend a service that itself extends another. You can extend indefinitely. Compose does not support circular references and `docker-compose` diff --git a/docs/extends.md b/docs/extends.md index c97b2b4fab..58def22d7f 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -10,16 +10,15 @@ weight=2 -## Extending services and Compose files +# Extending services and Compose files -Compose supports two ways to sharing common configuration and -extend a service with that shared configuration. +Compose supports two methods of sharing common configuration: 1. Extending individual services with [the `extends` field](#extending-services) 2. Extending entire compositions by - [exnteding compose files](#extending-compose-files) + [using multiple compose files](#multiple-compose-files) -### Extending services +## Extending services Docker Compose's `extends` keyword enables sharing of common configurations among different files, or even different projects entirely. Extending services @@ -30,9 +29,9 @@ place and refer to it from anywhere. > **Note:** `links` and `volumes_from` are never shared between services using > `extends`. See > [Adding and overriding configuration](#adding-and-overriding-configuration) -> for more information. + > for more information. -#### Understand the extends configuration +### Understand the extends configuration When defining any service in `docker-compose.yml`, you can declare that you are extending another service like this: @@ -54,8 +53,8 @@ looks like this: - "/data" In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with that `build`, `ports` and `volumes` configuration -defined directly under `web`. +`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration +values defined directly under `web`. You can go further and define (or re-define) configuration locally in `docker-compose.yml`: @@ -86,14 +85,14 @@ You can also write other services and link your `web` service to them: db: image: postgres -#### Example use case +### Example use case Extending an individual service is useful when you have multiple services that -have a common configuration. In this example we have a composition that with -a web application and a queue worker. Both services use the same codebase and -share many configuration options. +have a common configuration. The example below is a composition with +two services: a web application and a queue worker. Both services use the same +codebase and share many configuration options. -In a **common.yml** we'll define the common configuration: +In a **common.yml** we define the common configuration: app: build: . @@ -102,10 +101,9 @@ In a **common.yml** we'll define the common configuration: API_KEY: xxxyyy cpu_shares: 5 -In a **docker-compose.yml** we'll define the concrete services which use the +In a **docker-compose.yml** we define the concrete services which use the common configuration: - webapp: extends: file: common.yml @@ -121,11 +119,11 @@ common configuration: extends: file: common.yml service: app - command: /code/run_worker - links: - - queue + command: /code/run_worker + links: + - queue -#### Adding and overriding configuration +### Adding and overriding configuration Compose copies configurations from the original service over to the local one, **except** for `links` and `volumes_from`. These exceptions exist to avoid @@ -134,13 +132,11 @@ locally. This ensures dependencies between services are clearly visible when reading the current file. Defining these locally also ensures changes to the referenced file don't result in breakage. -If a configuration option is defined in both the original service and the local -service, the local value either *override*s or *extend*s the definition of the -original service. This works differently for other configuration options. +If a configuration option is defined in both the original service the local +service, the local value *replaces* or *extends* the original value. For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. **This is the default behaviour - all exceptions are -listed below.** +replaces the old value. # original service command: python app.py @@ -195,8 +191,8 @@ For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and - "4000" - "5000" -In the case of `environment` and `labels`, Compose "merges" entries together -with locally-defined values taking precedence: +In the case of `environment`, `labels`, `volumes` and `devices`, Compose +"merges" entries together with locally-defined values taking precedence: # original service environment: @@ -214,30 +210,154 @@ with locally-defined values taking precedence: - BAR=local - BAZ=local -Finally, for `volumes` and `devices`, Compose "merges" entries together with -locally-defined bindings taking precedence: - # original service - volumes: - - /original-dir/foo:/foo - - /original-dir/bar:/bar +## Multiple Compose files - # local service - volumes: - - /local-dir/bar:/bar - - /local-dir/baz/:baz +Using multiple Compose files enables you to customize a composition for +different environments or different workflows. - # result - volumes: - - /original-dir/foo:/foo - - /local-dir/bar:/bar - - /local-dir/baz/:baz +### Understanding multiple Compose files + +By default, Compose reads two files, a `docker-compose.yml` and an optional +`docker-compose.override.yml` file. By convention, the `docker-compose.yml` +contains your base configuration. The override file, as its name implies, can +contain configuration overrides for existing services or entirely new +services. + +If a service is defined in both files, Compose merges the configurations using +the same rules as the `extends` field (see [Adding and overriding +configuration](#adding-and-overriding-configuration)), with one exception. If a +service contains `links` or `volumes_from` those fields are copied over and +replace any values in the original service, in the same way single-valued fields +are copied. + +To use multiple override files, or an override file with a different name, you +can use the `-f` option to specify the list of files. Compose merges files in +the order they're specified on the command line. See the [`docker-compose` +command reference](./reference/docker-compose.md) for more information about +using `-f`. + +When you use multiple configuration files, you must make sure all paths in the +files are relative to the base Compose file (the first Compose file specified +with `-f`). This is required because override files need not be valid +Compose files. Override files can contain small fragments of configuration. +Tracking which fragment of a service is relative to which path is difficult and +confusing, so to keep paths easier to understand, all paths must be defined +relative to the base file. + +### Example use case + +In this section are two common use cases for multiple compose files: changing a +composition for different environments, and running administrative tasks +against a composition. + +#### Different environments + +A common use case for multiple files is changing a development composition +for a production-like environment (which may be production, staging or CI). +To support these differences, you can split your Compose configuration into +a few different files: + +Start with a base file that defines the canonical configuration for the +services. + +**docker-compose.yml** + + web: + image: example/my_web_app:latest + links: + - db + - cache + + db: + image: postgres:latest + + cache: + image: redis:latest + +In this example the development configuration exposes some ports to the +host, mounts our code as a volume, and builds the web image. + +**docker-compose.override.yml** -### Extending Compose files + web: + build: . + volumes: + - '.:/code' + ports: + - 8883:80 + environment: + DEBUG: 'true' -> **Note:** This feature is new in `docker-compose` 1.5 + db: + command: '-d' + ports: + - 5432:5432 + cache: + ports: + - 6379:6379 + +When you run `docker-compose up` it reads the overrides automatically. + +Now, it would be nice to use this composition in a production environment. So, +create another override file (which might be stored in a different git +repo or managed by a different team). + +**docker-compose.prod.yml** + + web: + ports: + - 80:80 + environment: + PRODUCTION: 'true' + + cache: + environment: + TTL: '500' + +To deploy with this production Compose file you can run + + docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d + +This deploys all three services using the configuration in +`docker-compose.yml` and `docker-compose.prod.yml` (but not the +dev configuration in `docker-compose.override.yml`). + + +See [production](production.md) for more information about Compose in +production. + +#### Administrative tasks + +Another common use case is running adhoc or administrative tasks against one +or more services in a composition. This example demonstrates running a +database backup. + +Start with a **docker-compose.yml**. + + web: + image: example/my_web_app:latest + links: + - db + + db: + image: postgres:latest + +In a **docker-compose.admin.yml** add a new service to run the database +export or backup. + + dbadmin: + build: database_admin/ + links: + - db + +To start a normal environment run `docker-compose up -d`. To run a database +backup, include the `docker-compose.admin.yml` as well. + + docker-compose -f docker-compose.yml -f docker-compose.admin.yml \ + run dbadmin db-backup ## Compose documentation diff --git a/docs/production.md b/docs/production.md index 39f0e1fe19..0a5e77b522 100644 --- a/docs/production.md +++ b/docs/production.md @@ -39,6 +39,9 @@ Once you've got a second configuration file, tell Compose to use it with the $ docker-compose -f docker-compose.yml -f production.yml up -d +See [Using multiple compose files](extends.md#different-environments) for a more +complete example. + ### Deploying changes When you make changes to your app code, you'll need to rebuild your image and From 62ebdce5a90ced63c27affcdf286189c11ea7885 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 15:14:42 -0500 Subject: [PATCH 335/337] Replace composition with Compose app. Signed-off-by: Daniel Nephin --- docs/extends.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index 58def22d7f..e4d09af98d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -15,8 +15,8 @@ weight=2 Compose supports two methods of sharing common configuration: 1. Extending individual services with [the `extends` field](#extending-services) -2. Extending entire compositions by - [using multiple compose files](#multiple-compose-files) +2. Extending entire Compose file by + [using multiple Compose files](#multiple-compose-files) ## Extending services @@ -88,7 +88,7 @@ You can also write other services and link your `web` service to them: ### Example use case Extending an individual service is useful when you have multiple services that -have a common configuration. The example below is a composition with +have a common configuration. The example below is a Compose app with two services: a web application and a queue worker. Both services use the same codebase and share many configuration options. @@ -213,8 +213,8 @@ In the case of `environment`, `labels`, `volumes` and `devices`, Compose ## Multiple Compose files -Using multiple Compose files enables you to customize a composition for -different environments or different workflows. +Using multiple Compose files enables you to customize a Compose application +for different environments or different workflows. ### Understanding multiple Compose files @@ -248,12 +248,12 @@ relative to the base file. ### Example use case In this section are two common use cases for multiple compose files: changing a -composition for different environments, and running administrative tasks -against a composition. +Compose app for different environments, and running administrative tasks +against a Compose app. #### Different environments -A common use case for multiple files is changing a development composition +A common use case for multiple files is changing a development Compose app for a production-like environment (which may be production, staging or CI). To support these differences, you can split your Compose configuration into a few different files: @@ -301,7 +301,7 @@ host, mounts our code as a volume, and builds the web image. When you run `docker-compose up` it reads the overrides automatically. -Now, it would be nice to use this composition in a production environment. So, +Now, it would be nice to use this Compose app in a production environment. So, create another override file (which might be stored in a different git repo or managed by a different team). @@ -332,7 +332,7 @@ production. #### Administrative tasks Another common use case is running adhoc or administrative tasks against one -or more services in a composition. This example demonstrates running a +or more services in a Compose app. This example demonstrates running a database backup. Start with a **docker-compose.yml**. From 40341674bd406c74eab83099cdc50684da3f45d4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 11:52:44 -0500 Subject: [PATCH 336/337] Re-order extends docs. Signed-off-by: Daniel Nephin --- docs/extends.md | 391 ++++++++++++++++++++++++------------------------ 1 file changed, 197 insertions(+), 194 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index e4d09af98d..b21b6d76db 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -14,201 +14,9 @@ weight=2 Compose supports two methods of sharing common configuration: -1. Extending individual services with [the `extends` field](#extending-services) -2. Extending entire Compose file by +1. Extending an entire Compose file by [using multiple Compose files](#multiple-compose-files) - -## Extending services - -Docker Compose's `extends` keyword enables sharing of common configurations -among different files, or even different projects entirely. Extending services -is useful if you have several services that reuse a common set of configuration -options. Using `extends` you can define a common set of service options in one -place and refer to it from anywhere. - -> **Note:** `links` and `volumes_from` are never shared between services using -> `extends`. See -> [Adding and overriding configuration](#adding-and-overriding-configuration) - > for more information. - -### Understand the extends configuration - -When defining any service in `docker-compose.yml`, you can declare that you are -extending another service like this: - - web: - extends: - file: common-services.yml - service: webapp - -This instructs Compose to re-use the configuration for the `webapp` service -defined in the `common-services.yml` file. Suppose that `common-services.yml` -looks like this: - - webapp: - build: . - ports: - - "8000:8000" - volumes: - - "/data" - -In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration -values defined directly under `web`. - -You can go further and define (or re-define) configuration locally in -`docker-compose.yml`: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - - important_web: - extends: web - cpu_shares: 10 - -You can also write other services and link your `web` service to them: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - links: - - db - db: - image: postgres - -### Example use case - -Extending an individual service is useful when you have multiple services that -have a common configuration. The example below is a Compose app with -two services: a web application and a queue worker. Both services use the same -codebase and share many configuration options. - -In a **common.yml** we define the common configuration: - - app: - build: . - environment: - CONFIG_FILE_PATH: /code/config - API_KEY: xxxyyy - cpu_shares: 5 - -In a **docker-compose.yml** we define the concrete services which use the -common configuration: - - webapp: - extends: - file: common.yml - service: app - command: /code/run_web_app - ports: - - 8080:8080 - links: - - queue - - db - - queue_worker: - extends: - file: common.yml - service: app - command: /code/run_worker - links: - - queue - -### Adding and overriding configuration - -Compose copies configurations from the original service over to the local one, -**except** for `links` and `volumes_from`. These exceptions exist to avoid -implicit dependencies—you always define `links` and `volumes_from` -locally. This ensures dependencies between services are clearly visible when -reading the current file. Defining these locally also ensures changes to the -referenced file don't result in breakage. - -If a configuration option is defined in both the original service the local -service, the local value *replaces* or *extends* the original value. - -For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. - - # original service - command: python app.py - - # local service - command: python otherapp.py - - # result - command: python otherapp.py - -In the case of `build` and `image`, using one in the local service causes -Compose to discard the other, if it was defined in the original service. - -Example of image replacing build: - - # original service - build: . - - # local service - image: redis - - # result - image: redis - - -Example of build replacing image: - - # original service - image: redis - - # local service - build: . - - # result - build: . - -For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and -`dns_search`, Compose concatenates both sets of values: - - # original service - expose: - - "3000" - - # local service - expose: - - "4000" - - "5000" - - # result - expose: - - "3000" - - "4000" - - "5000" - -In the case of `environment`, `labels`, `volumes` and `devices`, Compose -"merges" entries together with locally-defined values taking precedence: - - # original service - environment: - - FOO=original - - BAR=original - - # local service - environment: - - BAR=local - - BAZ=local - - # result - environment: - - FOO=original - - BAR=local - - BAZ=local +2. Extending individual services with [the `extends` field](#extending-services) ## Multiple Compose files @@ -360,6 +168,201 @@ backup, include the `docker-compose.admin.yml` as well. run dbadmin db-backup +## Extending services + +Docker Compose's `extends` keyword enables sharing of common configurations +among different files, or even different projects entirely. Extending services +is useful if you have several services that reuse a common set of configuration +options. Using `extends` you can define a common set of service options in one +place and refer to it from anywhere. + +> **Note:** `links` and `volumes_from` are never shared between services using +> `extends`. See +> [Adding and overriding configuration](#adding-and-overriding-configuration) + > for more information. + +### Understand the extends configuration + +When defining any service in `docker-compose.yml`, you can declare that you are +extending another service like this: + + web: + extends: + file: common-services.yml + service: webapp + +This instructs Compose to re-use the configuration for the `webapp` service +defined in the `common-services.yml` file. Suppose that `common-services.yml` +looks like this: + + webapp: + build: . + ports: + - "8000:8000" + volumes: + - "/data" + +In this case, you'll get exactly the same result as if you wrote +`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration +values defined directly under `web`. + +You can go further and define (or re-define) configuration locally in +`docker-compose.yml`: + + web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + + important_web: + extends: web + cpu_shares: 10 + +You can also write other services and link your `web` service to them: + + web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + links: + - db + db: + image: postgres + +### Example use case + +Extending an individual service is useful when you have multiple services that +have a common configuration. The example below is a Compose app with +two services: a web application and a queue worker. Both services use the same +codebase and share many configuration options. + +In a **common.yml** we define the common configuration: + + app: + build: . + environment: + CONFIG_FILE_PATH: /code/config + API_KEY: xxxyyy + cpu_shares: 5 + +In a **docker-compose.yml** we define the concrete services which use the +common configuration: + + webapp: + extends: + file: common.yml + service: app + command: /code/run_web_app + ports: + - 8080:8080 + links: + - queue + - db + + queue_worker: + extends: + file: common.yml + service: app + command: /code/run_worker + links: + - queue + +## Adding and overriding configuration + +Compose copies configurations from the original service over to the local one, +**except** for `links` and `volumes_from`. These exceptions exist to avoid +implicit dependencies—you always define `links` and `volumes_from` +locally. This ensures dependencies between services are clearly visible when +reading the current file. Defining these locally also ensures changes to the +referenced file don't result in breakage. + +If a configuration option is defined in both the original service the local +service, the local value *replaces* or *extends* the original value. + +For single-value options like `image`, `command` or `mem_limit`, the new value +replaces the old value. + + # original service + command: python app.py + + # local service + command: python otherapp.py + + # result + command: python otherapp.py + +In the case of `build` and `image`, using one in the local service causes +Compose to discard the other, if it was defined in the original service. + +Example of image replacing build: + + # original service + build: . + + # local service + image: redis + + # result + image: redis + + +Example of build replacing image: + + # original service + image: redis + + # local service + build: . + + # result + build: . + +For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and +`dns_search`, Compose concatenates both sets of values: + + # original service + expose: + - "3000" + + # local service + expose: + - "4000" + - "5000" + + # result + expose: + - "3000" + - "4000" + - "5000" + +In the case of `environment`, `labels`, `volumes` and `devices`, Compose +"merges" entries together with locally-defined values taking precedence: + + # original service + environment: + - FOO=original + - BAR=original + + # local service + environment: + - BAR=local + - BAZ=local + + # result + environment: + - FOO=original + - BAR=local + - BAZ=local + + + + ## Compose documentation - [User guide](/) From 77ff37a853b36e83de377cc83e7ff59eee11fc9c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 11:30:34 -0500 Subject: [PATCH 337/337] Bump 1.5.0 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 2 +- compose/__init__.py | 2 +- docs/install.md | 4 ++-- script/run.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0474ae2b3..a123c2a44d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.5.0 (2015-11-02) +1.5.0 (2015-11-03) ------------------ **Breaking changes:** diff --git a/compose/__init__.py b/compose/__init__.py index 7199babb40..2b8d5e72b2 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0rc3' +__version__ = '1.5.0' diff --git a/docs/install.md b/docs/install.md index 4eb0dc1869..c5304409c5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.0rc3 + docker-compose version: 1.5.0 ## Alternative install options @@ -76,7 +76,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc3/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 9ed1ea74cf..cf46c143c3 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0rc3" +VERSION="1.5.0" IMAGE="docker/compose:$VERSION"