FIX #2498 Introduce CI tests for external binary compatibility

Signed-off-by: David Gageot <david@gageot.net>

circle.yml

@@ -1,11 +1,9 @@
-# Pony-up!
 machine:
   pre:
   # Install gvm
     - bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/1.0.22/binscripts/gvm-installer)
   # Install bats
     - git clone https://github.com/sstephenson/bats.git && cd bats && sudo ./install.sh /usr/local
-    # - sudo apt-get install -y virtualbox
 
   post:
     - gvm install go1.5.2 -B --name=stable
@@ -16,8 +14,6 @@ machine:
     BASE_DIR: src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
   # Trick circle brainflat "no absolute path" behavior
     BASE_STABLE: ../../../$HOME/.gvm/pkgsets/stable/global/$BASE_DIR
-  # BASE_BLEED: ../../../$HOME/.gvm/pkgsets/bleed/global/$BASE_DIR
-    BUILDTAGS: ""
 
 dependencies:
   override:
@@ -31,23 +27,12 @@ test:
     - gvm use stable && go version
     - gvm use stable && make build:
         pwd: $BASE_STABLE
+    - gvm use stable && GO15VENDOREXPERIMENT=1 go get github.com/docker/docker-machine-driver-ci-test
 
   override:
     - DRIVER=none test/integration/run-bats.sh test/integration/cli:
         pwd: $BASE_STABLE
         timeout: 600
-    # - DRIVER=virtualbox test/integration/run-bats.sh test/integration/core:
-    #     pwd: $BASE_STABLE
-    #     timeout: 1200
-    # - DRIVER=digitalocean test/integration/run-bats.sh test/integration/core:
-    #     pwd: $BASE_STABLE
-    #     timeout: 1200
-    # - DRIVER=azure test/integration/run-bats.sh test/integration/core:
-    #     pwd: $BASE_STABLE
-    #     timeout: 1200
-    # - DRIVER=amazonec2 test/integration/run-bats.sh test/integration/core:
-    #     pwd: $BASE_STABLE
-    #     timeout: 1200
-
-    # - gvm use stable && DRIVER=amazonec2 test/integration/run-bats.sh test/integration/core:
-    #     pwd: $BASE_STABLE
+    - PATH=../../../../bin:$PATH DRIVER=ci-test test/integration/run-bats.sh test/integration/3rdparty:
+        pwd: $BASE_STABLE
+        timeout: 600

libmachine/libmachine.go

@@ -160,38 +160,39 @@ func (api *Client) performCreate(h *host.Host) error {
 		return fmt.Errorf("Error saving host to store after attempting creation: %s", err)
 	}
 
-	// TODO: Not really a fan of just checking "none" here.
-	if h.Driver.DriverName() != "none" {
-		log.Info("Waiting for machine to be running, this may take a few minutes...")
-		if err := mcnutils.WaitFor(drivers.MachineInState(h.Driver, state.Running)); err != nil {
-			return fmt.Errorf("Error waiting for machine to be running: %s", err)
-		}
-
-		log.Info("Machine is running, waiting for SSH to be available...")
-		if err := drivers.WaitForSSH(h.Driver); err != nil {
-			return fmt.Errorf("Error waiting for SSH: %s", err)
-		}
-
-		log.Info("Detecting operating system of created instance...")
-		provisioner, err := provision.DetectProvisioner(h.Driver)
-		if err != nil {
-			return fmt.Errorf("Error detecting OS: %s", err)
-		}
-
-		log.Infof("Provisioning with %s...", provisioner.String())
-		if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
-			return fmt.Errorf("Error running provisioning: %s", err)
-		}
-
-		// We should check the connection to docker here
-		log.Info("Checking connection to Docker...")
-		if _, _, err = check.DefaultConnChecker.Check(h, false); err != nil {
-			return fmt.Errorf("Error checking the host: %s", err)
-		}
-
-		log.Info("Docker is up and running!")
+	// TODO: Not really a fan of just checking "none" or "ci-test" here.
+	if h.Driver.DriverName() == "none" || h.Driver.DriverName() == "ci-test" {
+		return nil
 	}
 
+	log.Info("Waiting for machine to be running, this may take a few minutes...")
+	if err := mcnutils.WaitFor(drivers.MachineInState(h.Driver, state.Running)); err != nil {
+		return fmt.Errorf("Error waiting for machine to be running: %s", err)
+	}
+
+	log.Info("Machine is running, waiting for SSH to be available...")
+	if err := drivers.WaitForSSH(h.Driver); err != nil {
+		return fmt.Errorf("Error waiting for SSH: %s", err)
+	}
+
+	log.Info("Detecting operating system of created instance...")
+	provisioner, err := provision.DetectProvisioner(h.Driver)
+	if err != nil {
+		return fmt.Errorf("Error detecting OS: %s", err)
+	}
+
+	log.Infof("Provisioning with %s...", provisioner.String())
+	if err := provisioner.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions); err != nil {
+		return fmt.Errorf("Error running provisioning: %s", err)
+	}
+
+	// We should check the connection to docker here
+	log.Info("Checking connection to Docker...")
+	if _, _, err = check.DefaultConnChecker.Check(h, false); err != nil {
+		return fmt.Errorf("Error checking the host: %s", err)
+	}
+
+	log.Info("Docker is up and running!")
 	return nil
 }
 

test/integration/3rdparty/commands.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+load ${BASE_TEST_DIR}/helpers.bash
+
+only_if_env DRIVER ci-test
+
+@test "create" {
+  run machine create -d $DRIVER --url none default
+  [ "$status" -eq 0 ]
+}
+
+@test "ls" {
+  run machine ls -q
+  [ "$status" -eq 0 ]
+  [[ ${#lines[@]} == 1 ]]
+  [[ ${lines[0]} = "default" ]]
+}
+
+@test "url" {
+  run machine url default
+  [ "$status" -eq 0 ]
+  [[ ${output} == *"none"* ]]
+}
+
+@test "status" {
+  run machine status default
+  [ "$status" -eq 0 ]
+  [[ ${output} == *"Running"* ]]
+}
+
+@test "rm" {
+  run machine rm -y default
+  [ "$status" -eq 0 ]
+}