Merge pull request #123 from ehazlett/integration-tests

Integration tests

.travis.yml

@@ -5,4 +5,4 @@ go:
 script:
 - script/validate-dco
 - script/validate-gofmt
-- go test -v ./...
+- go test -v -short ./...

Dockerfile

@@ -1,6 +1,9 @@
 FROM golang:1.3-cross
+RUN apt-get update && apt-get install -y --no-install-recommends openssh-client
 RUN go get github.com/mitchellh/gox
 RUN go get github.com/aktau/github-release
+RUN go get github.com/tools/godep
 ENV GOPATH /go/src/github.com/docker/machine/Godeps/_workspace:/go
+ENV MACHINE_BINARY /go/src/github.com/docker/machine/machine
 WORKDIR /go/src/github.com/docker/machine
 ADD . /go/src/github.com/docker/machine

README.md

@@ -144,3 +144,15 @@ run:
 
 If you have any questions we're in #docker-machine on Freenode.
 
+=======
+## Integration Tests
+There is a suite of integration tests that will run for the drivers.  In order
+to use these you must export the corresponding environment variables for each
+driver as these perform the actual actions (start, stop, restart, kill, etc).
+
+By default, the suite will run tests against all drivers in master.  You can 
+override this by setting the environment variable `MACHINE_TESTS`.  For example,
+`MACHINE_TESTS="virtualbox" ./script/run-integration-tests` will only run the 
+virtualbox driver integration tests.
+
+To run, use the helper script `./script/run-integration-tests`.

_integration-test/machine_test.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"sync"
+	"testing"
+)
+
+const (
+	machineName = "machine-integration-test-%s"
+)
+
+func machineCreate(name string, t *testing.T, wg *sync.WaitGroup) {
+	mName := fmt.Sprintf(machineName, name)
+	fmt.Printf(" - testing create for %s (%s)\n", name, mName)
+	runCmd := exec.Command(machineBinary, "create", "-d", name, mName)
+	out, exitCode, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Error(out, err)
+	}
+	if exitCode != 0 {
+		t.Errorf("error creating machine: driver: %s; exit code: %d; output: %s", name, exitCode, out)
+	}
+	wg.Done()
+}
+
+func machineStop(name string, t *testing.T, wg *sync.WaitGroup) {
+	mName := fmt.Sprintf(machineName, name)
+	fmt.Printf(" - testing stop for %s (%s)\n", name, mName)
+	runCmd := exec.Command(machineBinary, "stop", mName)
+	out, exitCode, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Error(out, err)
+	}
+	if exitCode != 0 {
+		t.Errorf("error stopping machine: driver: %s; exit code: %d; output: %s", name, exitCode, out)
+	}
+	wg.Done()
+}
+
+func machineStart(name string, t *testing.T, wg *sync.WaitGroup) {
+	mName := fmt.Sprintf(machineName, name)
+	fmt.Printf(" - testing start for %s (%s)\n", name, mName)
+	runCmd := exec.Command(machineBinary, "start", mName)
+	out, exitCode, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Error(out, err)
+	}
+	if exitCode != 0 {
+		t.Errorf("error starting machine: driver: %s; exit code: %d; output: %s", name, exitCode, out)
+	}
+	wg.Done()
+}
+
+func machineKill(name string, t *testing.T, wg *sync.WaitGroup) {
+	mName := fmt.Sprintf(machineName, name)
+	fmt.Printf(" - testing kill for %s (%s)\n", name, mName)
+	runCmd := exec.Command(machineBinary, "kill", mName)
+	out, exitCode, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Error(out, err)
+	}
+	if exitCode != 0 {
+		t.Errorf("error killing machine: driver: %s; exit code: %d; output: %s", name, exitCode, out)
+	}
+	wg.Done()
+}
+
+func machineRm(name string, t *testing.T, wg *sync.WaitGroup) {
+	mName := fmt.Sprintf(machineName, name)
+	fmt.Printf(" - testing rm for %s (%s)\n", name, mName)
+	runCmd := exec.Command(machineBinary, "rm", "-f", mName)
+	out, exitCode, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Error(out, err)
+	}
+	if exitCode != 0 {
+		t.Errorf("error removing machine: driver: %s; exit code: %d; output: %s", name, exitCode, out)
+	}
+	wg.Done()
+}
+
+// TestMachineCreate will test that the driver creates the machine
+func TestMachineCreate(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	var wg sync.WaitGroup
+	for _, d := range machineTestDrivers {
+		wg.Add(1)
+		go machineCreate(d.name, t, &wg)
+	}
+	wg.Wait()
+}
+
+// TestMachineCreate will test that the driver stops the machine
+func TestMachineStop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	var wg sync.WaitGroup
+	for _, d := range machineTestDrivers {
+		wg.Add(1)
+		go machineStop(d.name, t, &wg)
+	}
+	wg.Wait()
+}
+
+// TestMachineCreate will test that the driver starts the machine
+func TestMachineStart(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	var wg sync.WaitGroup
+	for _, d := range machineTestDrivers {
+		wg.Add(1)
+		go machineStart(d.name, t, &wg)
+	}
+	wg.Wait()
+}
+
+// TestMachineCreate will test that the driver kills the machine
+func TestMachineKill(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	var wg sync.WaitGroup
+	for _, d := range machineTestDrivers {
+		wg.Add(1)
+		go machineKill(d.name, t, &wg)
+	}
+	wg.Wait()
+}
+
+// TestMachineCreate will test that the driver removes the machine
+func TestMachineRemove(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	var wg sync.WaitGroup
+	for _, d := range machineTestDrivers {
+		wg.Add(1)
+		go machineRm(d.name, t, &wg)
+	}
+	wg.Wait()
+}

_integration-test/test_vars.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type (
+	MachineDriver struct {
+		name string
+	}
+)
+
+var (
+	machineBinary      = "machine"
+	machineTestDrivers []MachineDriver
+)
+
+func init() {
+	// allow filtering driver tests
+	if machineTests := os.Getenv("MACHINE_TESTS"); machineTests != "" {
+		tests := strings.Split(machineTests, " ")
+		for _, test := range tests {
+			mcn := MachineDriver{
+				name: test,
+			}
+			machineTestDrivers = append(machineTestDrivers, mcn)
+		}
+	} else {
+		machineTestDrivers = []MachineDriver{
+			{
+				name: "virtualbox",
+			},
+			{
+				name: "digitalocean",
+			},
+		}
+	}
+
+	// find machine binary
+	if machineBin := os.Getenv("MACHINE_BINARY"); machineBin != "" {
+		machineBinary = machineBin
+	} else {
+		whichCmd := exec.Command("which", "machine")
+		out, _, err := runCommandWithOutput(whichCmd)
+		if err == nil {
+			machineBinary = stripTrailingCharacters(out)
+
+		} else {
+			fmt.Printf("ERROR: couldn't resolve full path to the Machine binary")
+			os.Exit(1)
+		}
+	}
+}

_integration-test/utils.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+func getExitCode(err error) (int, error) {
+	exitCode := 0
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		if procExit := exiterr.Sys().(syscall.WaitStatus); ok {
+			return procExit.ExitStatus(), nil
+		}
+	}
+	return exitCode, fmt.Errorf("failed to get exit code")
+}
+
+func processExitCode(err error) (exitCode int) {
+	if err != nil {
+		var exiterr error
+		if exitCode, exiterr = getExitCode(err); exiterr != nil {
+			// TODO: Fix this so we check the error's text.
+			// we've failed to retrieve exit code, so we set it to 127
+			exitCode = 127
+		}
+	}
+	return
+}
+
+func runCommandWithOutput(cmd *exec.Cmd) (output string, exitCode int, err error) {
+	exitCode = 0
+	out, err := cmd.CombinedOutput()
+	exitCode = processExitCode(err)
+	output = string(out)
+	return
+
+}
+
+func stripTrailingCharacters(target string) string {
+	target = strings.Trim(target, "\n")
+	target = strings.Trim(target, " ")
+	return target
+}

commands.go

@@ -95,7 +95,7 @@ var Commands = []cli.Command{
 
 			if name == "" {
 				cli.ShowCommandHelp(c, "create")
-				os.Exit(1)
+				log.Fatal("You must specify a machine name")
 			}
 
 			keyExists, err := drivers.PublicKeyExists()
@@ -257,7 +257,7 @@ var Commands = []cli.Command{
 		Action: func(c *cli.Context) {
 			if len(c.Args()) == 0 {
 				cli.ShowCommandHelp(c, "rm")
-				os.Exit(1)
+				log.Fatal("You must specify a machine name")
 			}
 
 			force := c.Bool("force")

script/run-integration-tests

@@ -0,0 +1,3 @@
+#!/bin/sh
+go test -v ./_integration-test
+

script/test

@@ -1,4 +1,4 @@
 #!/bin/sh
 set -e
 docker build -t docker-machine .
-exec docker run --rm docker-machine go test -v ./...
+exec docker run --rm docker-machine go test -v -short ./...