Merge pull request #2511 from dgageot/faster-integration-tests

Faster integration tests

test/integration/core/arbitrary-engine-envs.bats

@@ -1,19 +0,0 @@
-#!/usr/bin/env bats
-
-load ${BASE_TEST_DIR}/helpers.bash
-
-@test "$DRIVER: create with arbitrary engine envs" {
-  run machine create -d $DRIVER \
-    --engine-env=TEST=VALUE \
-    $NAME
-  echo ${output}
-  [ $status -eq 0  ]
-}
-
-@test "$DRIVER: test docker process envs" {
-  # get pid of docker process, check process envs for set Environment Variable from above test
-  run machine ssh $NAME 'sudo cat /proc/$(pgrep -f "docker [d]aemon")/environ'
-  echo ${output}
-  [ $status -eq 0 ]
-  [[ "${output}" =~ "TEST=VALUE" ]]
-}

test/integration/core/arbitrary-engine-options.bats

@@ -1,18 +0,0 @@
-#!/usr/bin/env bats
-
-load ${BASE_TEST_DIR}/helpers.bash
-
-@test "$DRIVER: create with arbitrary engine option" {
-  run machine create -d $DRIVER \
-    --engine-opt log-driver=none \
-    $NAME
-  echo ${output}
-  [ $status -eq 0 ]
-}
-
-@test "$DRIVER: check created engine option (log driver)" {
-  docker $(machine config $NAME) run --name nolog busybox echo this should not be logged
-  run docker $(machine config $NAME) inspect -f '{{.HostConfig.LogConfig.Type}}' nolog
-  echo ${output}
-  [ ${output} == "none" ]
-}

test/integration/core/certs-extra-san.bats

@@ -2,6 +2,7 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
+use_disposable_machine
 
 @test "$DRIVER: create" {
   run machine create --tls-san foo.bar.tld --tls-san 10.42.42.42  -d $DRIVER $NAME

test/integration/core/core-commands.bats

@@ -2,17 +2,13 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
+use_shared_machine
+
 @test "$DRIVER: machine should not exist" {
-  run machine inspect $NAME
+  run machine inspect UNKNOWN
   echo ${output}
   [ "$status" -eq 1 ]
-  [[ ${lines[0]} =~ "Host does not exist: \"$NAME\"" ]]
+  [[ ${lines[0]} =~ "Host does not exist: \"UNKNOWN\"" ]]
-}
-
-@test "$DRIVER: create" {
-  run machine create -d $DRIVER $NAME
-  echo ${output}
-  [ "$status" -eq 0  ]
 }
 
 @test "$DRIVER: appears with ls" {

test/integration/core/engine-options.bats

@@ -0,0 +1,42 @@
+#!/usr/bin/env bats
+
+load ${BASE_TEST_DIR}/helpers.bash
+
+use_disposable_machine
+
+@test "$DRIVER: create with supported engine options" {
+  run machine create -d $DRIVER \
+    --engine-label spam=eggs \
+    --engine-storage-driver overlay \
+    --engine-insecure-registry registry.myco.com \
+    --engine-env=TEST=VALUE \
+    --engine-opt log-driver=none \
+    $NAME
+  echo "$output"
+  [ $status -eq 0 ]
+}
+
+@test "$DRIVER: check for engine label" {
+  spamlabel=$(docker $(machine config $NAME) info | grep spam)
+  [[ $spamlabel =~ "spam=eggs" ]]
+}
+
+@test "$DRIVER: check for engine storage driver" {
+  storage_driver_info=$(docker $(machine config $NAME) info | grep "Storage Driver")
+  [[ $storage_driver_info =~ "overlay" ]]
+}
+
+@test "$DRIVER: test docker process envs" {
+  # get pid of docker process, check process envs for set Environment Variable from above test
+  run machine ssh $NAME 'sudo cat /proc/$(pgrep -f "docker [d]aemon")/environ'
+  echo ${output}
+  [ $status -eq 0 ]
+  [[ "${output}" =~ "TEST=VALUE" ]]
+}
+
+@test "$DRIVER: check created engine option (log driver)" {
+  docker $(machine config $NAME) run --name nolog busybox echo this should not be logged
+  run docker $(machine config $NAME) inspect -f '{{.HostConfig.LogConfig.Type}}' nolog
+  echo ${output}
+  [ ${output} == "none" ]
+}

test/integration/core/env_shell.bats

@@ -2,10 +2,7 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
-@test "$DRIVER: create" {
+use_shared_machine
-  run machine create -d $DRIVER $NAME
-  [ "$status" -eq 0  ]
-}
 
 @test "$DRIVER: test basic bash / zsh notation" {
   run machine env $NAME

test/integration/core/inspect_format.bats

@@ -2,10 +2,7 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
-@test "$DRIVER: create" {
+use_shared_machine
-  run machine create -d $DRIVER $NAME
-  [ "$status" -eq 0  ]
-}
 
 @test "$DRIVER: inspect format template" {
   run machine inspect -f '{{.DriverName}}' $NAME
@@ -24,7 +21,7 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 @test "$DRIVER: check .Driver output is not flawed" {
   only_if_env DRIVER virtualbox
-  run docker-machine inspect -f '{{.Driver.SSHUser}}' $NAME
+  run machine inspect -f '{{.Driver.SSHUser}}' $NAME
   [ "$status" -eq 0 ]
   [[ ${output} == "docker" ]]
 }

test/integration/core/regenerate-certs.bats

@@ -2,9 +2,7 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
-@test "$DRIVER: create" {
+use_shared_machine
-  run machine create -d $DRIVER $NAME
-}
 
 @test "$DRIVER: regenerate the certs" {
   run machine regenerate-certs -f $NAME

test/integration/core/scp.bats

@@ -2,13 +2,9 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
+use_shared_machine
 export SECOND_MACHINE="$NAME-2"
 
-@test "$DRIVER: create" {
-  run machine create -d $DRIVER $NAME
-  [[ ${status} -eq 0 ]]
-}
-
 @test "$DRIVER: test machine scp command from remote to host" {
   machine ssh $NAME 'echo A file created remotely! >/tmp/foo.txt'
   machine scp $NAME:/tmp/foo.txt .

test/integration/core/ssh-backends.bats

@@ -2,12 +2,7 @@
 
 load ${BASE_TEST_DIR}/helpers.bash
 
-# Basic smoke test for SSH backends
+use_shared_machine
-
-@test "$DRIVER: create SSH test box" {
-  run machine create -d $DRIVER $NAME
-  [[ "$status" -eq 0  ]]
-}
 
 @test "$DRIVER: test external ssh backend" {
   run machine ssh $NAME df -h

test/integration/core/supported-engine-options.bats

@@ -1,23 +0,0 @@
-#!/usr/bin/env bats
-
-load ${BASE_TEST_DIR}/helpers.bash
-
-@test "$DRIVER: create with supported engine options" {
-  run machine create -d $DRIVER \
-    --engine-label spam=eggs \
-    --engine-storage-driver overlay \
-    --engine-insecure-registry registry.myco.com \
-    $NAME
-  echo "$output"
-  [ $status -eq 0 ]
-}
-
-@test "$DRIVER: check for engine label" {
-  spamlabel=$(docker $(machine config $NAME) info | grep spam)
-  [[ $spamlabel =~ "spam=eggs" ]]
-}
-
-@test "$DRIVER: check for engine storage driver" {
-  storage_driver_info=$(docker $(machine config $NAME) info | grep "Storage Driver")
-  [[ $storage_driver_info =~ "overlay" ]]
-}

test/integration/core/swarm-options.bats

@@ -1,7 +1,8 @@
 #!/usr/bin/env bats
 
 load ${BASE_TEST_DIR}/helpers.bash
-export TOKEN=$(curl -sS -X POST "https://discovery-stage.hub.docker.com/v1/clusters")
+
+export TOKEN=$(curl -sS -X POST "https://discovery.hub.docker.com/v1/clusters")
 
 @test "create swarm master" {
     run machine create -d $DRIVER --swarm --swarm-master --swarm-discovery "token://$TOKEN" --swarm-strategy binpack --swarm-opt heartbeat=5s queenbee
@@ -21,7 +22,7 @@ export TOKEN=$(curl -sS -X POST "https://discovery-stage.hub.docker.com/v1/clust
 }
 
 @test "ensure heartbeat" {
-    heartbeat_arg=$(docker $(machine config queenbee) inspect -f '{{index .Args 9}}' swarm-agent-master)
+    heartbeat_arg=$(docker $(machine config queenbee) inspect -f '{{index .Args}}' swarm-agent-master)
     echo ${heartbeat_arg}
-    [[ "$heartbeat_arg" == "--heartbeat=5s" ]]
+    [[ "$heartbeat_arg" =~ "--heartbeat=5s" ]]
 }

test/integration/helpers.bash

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo_to_log() {
+function echo_to_log {
     echo "$BATS_TEST_NAME
 ----------
 $output
@@ -9,27 +9,42 @@ $output
 "   >> ${BATS_LOG}
 }
 
-teardown() {
+function teardown {
     echo_to_log
 }
 
-function errecho () {
+function errecho {
     >&2 echo "$@"
 }
 
-function only_if_env () {
+function only_if_env {
     if [[ ${!1} != "$2" ]]; then
         errecho "This test requires the $1 environment variable to be set to $2. Skipping..."
         skip
     fi
 }
 
-function require_env () {
+function require_env {
     if [[ -z ${!1} ]]; then
         errecho "This test requires the $1 environment variable to be set in order to run."
         exit 1
     fi
 }
 
+function use_disposable_machine {
+    if [[ -z "$NAME" ]]; then
+        export NAME="bats-$DRIVER-test-$(date +%s)"
+    fi
+}
+
+function use_shared_machine {
+    if [[ -z "$NAME" ]]; then
+      export NAME="$SHARED_NAME"
+      if [[ $(machine ls -q --filter name=$NAME | wc -l) -eq 0 ]]; then
+          machine create -d $DRIVER $NAME &>/dev/null
+      fi
+    fi
+}
+
 # Make sure these aren't set while tests run (can cause confusing behavior)
 unset DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_DIR

test/integration/run-bats.sh

@@ -14,9 +14,11 @@ function quiet_run () {
 }
 
 function cleanup_machines() {
-    if [[ $(machine ls -q | wc -l) -ne 0 ]]; then
+    for MACHINE_NAME in $(machine ls -q); do
-        quiet_run machine rm -f $(machine ls -q)
+        if [[ "$MACHINE_NAME" != "$SHARED_NAME" ]] || [[ "$1" == "ALL" ]]; then
-    fi
+            quiet_run machine rm -f $MACHINE_NAME
+        fi
+    done
 }
 
 function cleanup_store() {
@@ -26,18 +28,15 @@ function cleanup_store() {
 }
 
 function machine() {
-    export PATH="$MACHINE_ROOT"/bin:$PATH
     "$MACHINE_ROOT"/bin/"$MACHINE_BIN_NAME" "$@"
 }
 
 function run_bats() {
     for bats_file in $(find "$1" -name \*.bats); do
-        export NAME="bats-$DRIVER-test-$(date +%s)"
-
-        # BATS returns non-zero to indicate the tests have failed, we shouldn't
-        # neccesarily bail in this case, so that's the reason for the e toggle.
         echo "=> $bats_file"
 
+        # BATS returns non-zero to indicate the tests have failed, we shouldn't
+        # necessarily bail in this case, so that's the reason for the e toggle.
         set +e
         bats "$bats_file"
         if [[ $? -ne 0 ]]; then
@@ -46,7 +45,12 @@ function run_bats() {
         set -e
 
         echo
-        cleanup_machines
+
+        if [[ "$NO_SHARE_MACHINES" == "1" ]]; then
+            cleanup_machines "ALL"
+        else
+            cleanup_machines "NON-SHARED"
+        fi
     done
 }
 
@@ -88,13 +92,14 @@ export MACHINE_STORAGE_PATH="/tmp/machine-bats-test-$DRIVER"
 export MACHINE_BIN_NAME=docker-machine
 export BATS_LOG="$MACHINE_ROOT/bats.log"
 export B2D_LOCATION=~/.docker/machine/cache/boot2docker.iso
+export SHARED_NAME="bats-$DRIVER-test-shared-$(date +%s)"
 
 # This function gets used in the integration tests, so export it.
 export -f machine
 
 > "$BATS_LOG"
 
-cleanup_machines
+cleanup_machines "ALL"
 cleanup_store
 
 if [[ -f "$B2D_LOCATION" ]]; then
@@ -108,6 +113,7 @@ fi
 
 run_bats "$BATS_FILE"
 
+cleanup_machines "ALL"
 cleanup_store
 
 exit ${EXIT_STATUS}

test/integration/virtualbox/bad-create-iso.bats

@@ -4,6 +4,8 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
+use_disposable_machine
+
 export BAD_URL="http://dev.null:9111/bad.iso"
 
 @test "$DRIVER: Should not allow machine creation with bad ISO" {

test/integration/virtualbox/certs-checksum.bats

@@ -4,9 +4,7 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
-@test "$DRIVER: create" {
+use_shared_machine
-  run machine create -d $DRIVER $NAME
-}
 
 @test "$DRIVER: verify that server cert checksum matches local checksum" {
   # TODO: This test is tightly coupled to VirtualBox right now, but should be

test/integration/virtualbox/custom-mem-disk.bats

@@ -4,6 +4,8 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
+use_disposable_machine
+
 # Default memsize is 1024MB and disksize is 20000MB
 # These values are defined in drivers/virtualbox/virtualbox.go
 export DEFAULT_MEMSIZE=1024

test/integration/virtualbox/dns.bats

@@ -4,6 +4,8 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
+use_disposable_machine
+
 @test "$DRIVER: Create a vm with a dns proxy set" {
   run machine create -d $DRIVER --virtualbox-dns-proxy=true $NAME
   [[ ${status} -eq 0 ]]

test/integration/virtualbox/pause-save-start.bats

@@ -4,10 +4,7 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
-@test "$DRIVER: create" {
+use_shared_machine
-  run machine create -d $DRIVER $NAME
-  [ "$status" -eq 0  ]
-}
 
 @test "$DRIVER: VBoxManage pause" {
   run VBoxManage controlvm $NAME pause

test/integration/virtualbox/upgrade.bats

@@ -4,6 +4,8 @@ load ${BASE_TEST_DIR}/helpers.bash
 
 only_if_env DRIVER virtualbox
 
+use_disposable_machine
+
 export OLD_ISO_URL="https://github.com/boot2docker/boot2docker/releases/download/v1.4.1/boot2docker.iso"
 
 @test "$DRIVER: create for upgrade" {