From 158e3068bda22ee09b00d37b3e4b33e42ecced1b Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 27 Jun 2014 15:55:36 -0400 Subject: [PATCH 1/6] compile unit tests in parallel with GNU parallel Docker-DCO-1.1-Signed-off-by: Tibor Vass (github: tiborvass) --- hack/make.sh | 19 +++++++++++ hack/make/test-unit | 77 ++++++++++++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index 843c84b1f4..e93fa2d5b1 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -156,6 +156,25 @@ go_test_dir() { ) } +# Compile phase run by parallel in test-unit. No support for coverpkg +go_compile_test_dir() { + dir=$1 + testcover=() + if [ "$HAVE_GO_TEST_COVER" ]; then + # if our current go install has -cover, we want to use it :) + mkdir -p "$DEST/coverprofiles" + coverprofile="docker${dir#.}" + coverprofile="$DEST/coverprofiles/${coverprofile//\//-}" + testcover=( -cover -coverprofile "$coverprofile" ) # missing $coverpkg + fi + ( + readarray -t BUILDFLAGS < "$BUILDFLAGS_FILE" + cd "$dir" + go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c + echo "$dir" + ) +} + # This helper function walks the current directory looking for directories # holding certain files ($1 parameter), and prints their paths on standard # output, one per line. diff --git a/hack/make/test-unit b/hack/make/test-unit index 552810f349..0ec945a1ed 100644 --- a/hack/make/test-unit +++ b/hack/make/test-unit @@ -6,6 +6,7 @@ DEST=$1 RED=$'\033[31m' GREEN=$'\033[32m' TEXTRESET=$'\033[0m' # reset the foreground colour +: ${PARALLEL_JOBS:=0} # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. @@ -22,35 +23,59 @@ bundle_test_unit() { TESTDIRS=$(find_dirs '*_test.go') fi - TESTS_FAILED=() - for test_dir in $TESTDIRS; do - echo + ( + # accomodate parallel to be able to access variables + export SHELL=/bin/bash + export HOME=/tmp + mkdir -p "$HOME/.parallel" + touch "$HOME/.parallel/ignored_vars" + export -f go_compile_test_dir + export LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER" + export TESTFLAGS + export HAVE_GO_TEST_COVER + export DEST + # some hack to export array variables + export BUILDFLAGS_FILE="$HOME/buildflags_file" + ( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE" - if ! LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER" go_test_dir "$test_dir"; then - TESTS_FAILED+=("$test_dir") - echo - echo "${RED}Tests failed: $test_dir${TEXTRESET}" - sleep 1 # give it a second, so observers watching can take note - fi - done - - echo - echo - echo - - # if some tests fail, we want the bundlescript to fail, but we want to - # try running ALL the tests first, hence TESTS_FAILED - if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then - echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" - echo - false - else - echo "${GREEN}Test success${TEXTRESET}" - echo - true - fi + echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir | go_run_test_dir + ) } } +go_run_test_dir() { + TESTS_FAILED=() + while read dir; do + echo + echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}" + pushd "$dir" > /dev/null + t=$(basename "$dir").test + if ! ./$t; then + TESTS_FAILED+=("$dir") + echo + echo "${RED}Tests failed: $dir${TEXTRESET}" + sleep 1 # give it a second, so observers watching can take note + fi + rm ./$t || true + popd > /dev/null + done + + echo + echo + echo + + # if some tests fail, we want the bundlescript to fail, but we want to + # try running ALL the tests first, hence TESTS_FAILED + if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then + echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}" + echo + false + else + echo "${GREEN}Test success${TEXTRESET}" + echo + true + fi +} + exec > >(tee -a $DEST/test.log) 2>&1 bundle_test_unit From 3a1385702cdd69a0b85b726dac6e749eb0e97fc1 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 3 Jul 2014 14:29:07 -0400 Subject: [PATCH 2/6] benign whitespace change Docker-DCO-1.1-Signed-off-by: Tibor Vass (github: tiborvass) --- hack/make.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index e93fa2d5b1..93a3ba9c8c 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -42,17 +42,17 @@ echo DEFAULT_BUNDLES=( validate-dco validate-gofmt - + binary - + test-unit test-integration test-integration-cli - + dynbinary dyntest-unit dyntest-integration - + cover cross tgz From a6085fd430e8f155e4a0738de752df3c77bc132f Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 3 Jul 2014 15:11:32 -0600 Subject: [PATCH 3/6] Add a bunch more tweaks to the parallel test compilation - put all the precompiled test binaries in $DEST so they show up in bundles and can be re-run individually afterwards - support cases where parallel is not installed (when using dyntest-unit, for example, this is much more common, since it's designed to be run outside the Dockerfile) - use "mktemp -d" instead of "/tmp" directly for our temporary parallel HOME - update the default PARALLEL_JOBS to be the value of "nproc" instead of 0, since "0 means as many as possible" (see https://www.gnu.org/software/parallel/man.html#jobs_n) Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- hack/make.sh | 9 +++++++-- hack/make/test-unit | 26 +++++++++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index 93a3ba9c8c..4c7dea4542 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -159,6 +159,7 @@ go_test_dir() { # Compile phase run by parallel in test-unit. No support for coverpkg go_compile_test_dir() { dir=$1 + out_file="$DEST/precompiled/$dir.test" testcover=() if [ "$HAVE_GO_TEST_COVER" ]; then # if our current go install has -cover, we want to use it :) @@ -167,12 +168,16 @@ go_compile_test_dir() { coverprofile="$DEST/coverprofiles/${coverprofile//\//-}" testcover=( -cover -coverprofile "$coverprofile" ) # missing $coverpkg fi - ( + if [ "$BUILDFLAGS_FILE" ]; then readarray -t BUILDFLAGS < "$BUILDFLAGS_FILE" + fi + ( cd "$dir" go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c - echo "$dir" ) + mkdir -p "$(dirname "$out_file")" + mv "$dir/$(basename "$dir").test" "$out_file" + echo "Precompiled: github.com/dotcloud/docker${dir#.}" } # This helper function walks the current directory looking for directories diff --git a/hack/make/test-unit b/hack/make/test-unit index 0ec945a1ed..0666a6c82a 100644 --- a/hack/make/test-unit +++ b/hack/make/test-unit @@ -2,11 +2,11 @@ set -e DEST=$1 +: ${PARALLEL_JOBS:=$(nproc)} RED=$'\033[31m' GREEN=$'\033[32m' TEXTRESET=$'\033[0m' # reset the foreground colour -: ${PARALLEL_JOBS:=0} # Run Docker's test suite, including sub-packages, and store their output as a bundle # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. @@ -23,10 +23,10 @@ bundle_test_unit() { TESTDIRS=$(find_dirs '*_test.go') fi - ( + if command -v parallel &> /dev/null; then ( # accomodate parallel to be able to access variables - export SHELL=/bin/bash - export HOME=/tmp + export SHELL="$BASH" + export HOME="$(mktemp -d)" mkdir -p "$HOME/.parallel" touch "$HOME/.parallel/ignored_vars" export -f go_compile_test_dir @@ -38,8 +38,15 @@ bundle_test_unit() { export BUILDFLAGS_FILE="$HOME/buildflags_file" ( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE" - echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir | go_run_test_dir - ) + echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir + rm -rf "$HOME" + ) else + # aww, no "parallel" available - fall back to boring + for test_dir in $TESTDIRS; do + go_compile_test_dir "$test_dir" + done + fi + echo "$TESTDIRS" | go_run_test_dir } } @@ -48,16 +55,13 @@ go_run_test_dir() { while read dir; do echo echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}" - pushd "$dir" > /dev/null - t=$(basename "$dir").test - if ! ./$t; then + precompiled="$DEST/precompiled/$dir.test" + if ! "$precompiled"; then TESTS_FAILED+=("$dir") echo echo "${RED}Tests failed: $dir${TEXTRESET}" sleep 1 # give it a second, so observers watching can take note fi - rm ./$t || true - popd > /dev/null done echo From 81e78db507c7523696b4115b5675170925a417e9 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 9 Jul 2014 17:21:39 -0400 Subject: [PATCH 4/6] remove temporary line in Dockerfile that prevented cache miss Docker-DCO-1.1-Signed-off-by: Tibor Vass (github: tiborvass) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5369962359..416f53c782 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \ lxc=1.0* \ mercurial \ pandoc \ + parallel \ reprepro \ ruby1.9.1 \ ruby1.9.1-dev \ From 0a7cd70924178304d1919ae5ea2cc86620f2be54 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 9 Jul 2014 16:43:57 -0600 Subject: [PATCH 5/6] Fix precompiled unit tests needing their "testdata" contents Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- hack/make/test-unit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/make/test-unit b/hack/make/test-unit index 0666a6c82a..91a853309b 100644 --- a/hack/make/test-unit +++ b/hack/make/test-unit @@ -56,7 +56,7 @@ go_run_test_dir() { echo echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}" precompiled="$DEST/precompiled/$dir.test" - if ! "$precompiled"; then + if ! ( cd "$dir" && "$precompiled" ); then TESTS_FAILED+=("$dir") echo echo "${RED}Tests failed: $dir${TEXTRESET}" From 4cfe806db2a41a97079a4881d4a0099a7b5b8149 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Sat, 26 Jul 2014 13:45:35 -0400 Subject: [PATCH 6/6] Kill all concurrent jobs on error Docker-DCO-1.1-Signed-off-by: Tibor Vass (github: tiborvass) --- hack/make.sh | 1 + hack/make/test-unit | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/make.sh b/hack/make.sh index 4c7dea4542..ca3c775bfd 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -175,6 +175,7 @@ go_compile_test_dir() { cd "$dir" go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c ) + [ $? -ne 0 ] && return 1 mkdir -p "$(dirname "$out_file")" mv "$dir/$(basename "$dir").test" "$out_file" echo "Precompiled: github.com/dotcloud/docker${dir#.}" diff --git a/hack/make/test-unit b/hack/make/test-unit index 91a853309b..1ac3f4aa5b 100644 --- a/hack/make/test-unit +++ b/hack/make/test-unit @@ -38,7 +38,7 @@ bundle_test_unit() { export BUILDFLAGS_FILE="$HOME/buildflags_file" ( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE" - echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir + echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --halt 2 --env _ go_compile_test_dir rm -rf "$HOME" ) else # aww, no "parallel" available - fall back to boring