diff --git a/README.md b/README.md index 812756345a..3c507e4bc9 100644 --- a/README.md +++ b/README.md @@ -122,16 +122,53 @@ the coverage for the VirtualBox driver's package, browse to `/drivers/virtualbox You can hit `CTRL+C` to stop the server. ## Integration Tests -There is a suite of integration tests that will run for the drivers. In order -to use these you must export the corresponding environment variables for each -driver as these perform the actual actions (start, stop, restart, kill, etc). +We utilize [BATS](https://github.com/sstephenson/bats) for integration testing. +This runs tests against the generated binary. To use, make sure to install +BATS (use that link). Then run `./script/build` to generate the binary. Once +you have the binary, you can run test against a specified driver: -By default, the suite will run tests against all drivers in master. You can -override this by setting the environment variable `MACHINE_TESTS`. For example, -`MACHINE_TESTS="virtualbox" ./script/run-integration-tests` will only run the -virtualbox driver integration tests. +``` +$ DRIVER=virtualbox bats test/driver.bats + ✓ virtualbox: machine should not exist + ✓ virtualbox: create + ✓ virtualbox: active + ✓ virtualbox: ls + ✓ virtualbox: url + ✓ virtualbox: ip + ✓ virtualbox: ssh + ✓ virtualbox: stop + ✓ virtualbox: machine should show stopped + ✓ virtualbox: start + ✓ virtualbox: machine should show running after start + ✓ virtualbox: restart + ✓ virtualbox: machine should show running after restart + ✓ virtualbox: remove + ✓ virtualbox: machine should not exist -You can set the path to the machine binary under test using the `MACHINE_BINARY` -environment variable. +15 tests, 0 failures +``` -To run, use the helper script `./script/run-integration-tests`. +You can also run the general `cli` tests: + +``` +$ bats test/cli.bats + ✓ cli: show info + ✓ cli: show active help + ✓ cli: show config help + ✓ cli: show inspect help + ✓ cli: show ip help + ✓ cli: show kill help + ✓ cli: show ls help + ✓ cli: show restart help + ✓ cli: show rm help + ✓ cli: show env help + ✓ cli: show ssh help + ✓ cli: show start help + ✓ cli: show stop help + ✓ cli: show upgrade help + ✓ cli: show url help + ✓ flag: show version + ✓ flag: show help + +17 tests, 0 failures +``` diff --git a/_integration-test/machine_test.go b/_integration-test/machine_test.go deleted file mode 100644 index c09fa0267a..0000000000 --- a/_integration-test/machine_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "fmt" - "os/exec" - "sync" - "testing" - "time" -) - -var ( - machineName = fmt.Sprintf("machine-test-%d", time.Now().UnixNano()) + "-%s" -) - -func machineCreate(name string, t *testing.T, wg *sync.WaitGroup) { - mName := fmt.Sprintf(machineName, name) - fmt.Printf(" - testing create for %s (%s)\n", name, mName) - runCmd := exec.Command(machineBinary, "create", "-d", name, mName) - out, exitCode, err := runCommandWithOutput(runCmd) - if err != nil { - t.Error(out, err) - } - if exitCode != 0 { - t.Errorf("error creating machine: driver: %s; exit code: %d; output: %s", name, exitCode, out) - } - wg.Done() -} - -func machineStop(name string, t *testing.T, wg *sync.WaitGroup) { - mName := fmt.Sprintf(machineName, name) - fmt.Printf(" - testing stop for %s (%s)\n", name, mName) - runCmd := exec.Command(machineBinary, "stop", mName) - out, exitCode, err := runCommandWithOutput(runCmd) - if err != nil { - t.Error(out, err) - } - if exitCode != 0 { - t.Errorf("error stopping machine: driver: %s; exit code: %d; output: %s", name, exitCode, out) - } - wg.Done() -} - -func machineStart(name string, t *testing.T, wg *sync.WaitGroup) { - mName := fmt.Sprintf(machineName, name) - fmt.Printf(" - testing start for %s (%s)\n", name, mName) - runCmd := exec.Command(machineBinary, "start", mName) - out, exitCode, err := runCommandWithOutput(runCmd) - if err != nil { - t.Error(out, err) - } - if exitCode != 0 { - t.Errorf("error starting machine: driver: %s; exit code: %d; output: %s", name, exitCode, out) - } - wg.Done() -} - -func machineKill(name string, t *testing.T, wg *sync.WaitGroup) { - mName := fmt.Sprintf(machineName, name) - fmt.Printf(" - testing kill for %s (%s)\n", name, mName) - runCmd := exec.Command(machineBinary, "kill", mName) - out, exitCode, err := runCommandWithOutput(runCmd) - if err != nil { - t.Error(out, err) - } - if exitCode != 0 { - t.Errorf("error killing machine: driver: %s; exit code: %d; output: %s", name, exitCode, out) - } - wg.Done() -} - -func machineRm(name string, t *testing.T, wg *sync.WaitGroup) { - mName := fmt.Sprintf(machineName, name) - fmt.Printf(" - testing rm for %s (%s)\n", name, mName) - runCmd := exec.Command(machineBinary, "rm", "-f", mName) - out, exitCode, err := runCommandWithOutput(runCmd) - if err != nil { - t.Error(out, err) - } - if exitCode != 0 { - t.Errorf("error removing machine: driver: %s; exit code: %d; output: %s", name, exitCode, out) - } - wg.Done() -} - -// TestMachineCreate will test that the driver creates the machine -func TestMachineCreate(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - var wg sync.WaitGroup - for _, d := range machineTestDrivers { - wg.Add(1) - go machineCreate(d.name, t, &wg) - } - wg.Wait() -} - -// TestMachineCreate will test that the driver stops the machine -func TestMachineStop(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - var wg sync.WaitGroup - for _, d := range machineTestDrivers { - wg.Add(1) - go machineStop(d.name, t, &wg) - } - wg.Wait() - time.Sleep(waitDuration) -} - -// TestMachineCreate will test that the driver starts the machine -func TestMachineStart(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - var wg sync.WaitGroup - for _, d := range machineTestDrivers { - wg.Add(1) - go machineStart(d.name, t, &wg) - } - wg.Wait() - time.Sleep(waitDuration) -} - -// TestMachineCreate will test that the driver kills the machine -func TestMachineKill(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - var wg sync.WaitGroup - for _, d := range machineTestDrivers { - wg.Add(1) - go machineKill(d.name, t, &wg) - } - wg.Wait() - time.Sleep(waitDuration) -} - -// TestMachineCreate will test that the driver removes the machine -func TestMachineRemove(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - var wg sync.WaitGroup - for _, d := range machineTestDrivers { - wg.Add(1) - go machineRm(d.name, t, &wg) - } - wg.Wait() - time.Sleep(waitDuration) -} diff --git a/_integration-test/test_vars.go b/_integration-test/test_vars.go deleted file mode 100644 index 6e86184c54..0000000000 --- a/_integration-test/test_vars.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/exec" - "strconv" - "strings" - "time" -) - -type ( - MachineDriver struct { - name string - } -) - -var ( - machineBinary = "machine" - machineTestDrivers []MachineDriver - waitInterval int - waitDuration time.Duration -) - -func init() { - // allow filtering driver tests - if machineTests := os.Getenv("MACHINE_TESTS"); machineTests != "" { - tests := strings.Split(machineTests, " ") - for _, test := range tests { - mcn := MachineDriver{ - name: test, - } - machineTestDrivers = append(machineTestDrivers, mcn) - } - } else { - machineTestDrivers = []MachineDriver{ - { - name: "virtualbox", - }, - { - name: "digitalocean", - }, - } - } - - interval := os.Getenv("MACHINE_TEST_DURATION") - if interval == "" { - interval = "30" - } - wait, err := strconv.Atoi(interval) - if err != nil { - fmt.Printf("invalid interval: %s\n", err) - os.Exit(1) - } - - waitInterval = wait - waitDuration = time.Duration(time.Duration(waitInterval) * time.Second) - - // find machine binary - if machineBin := os.Getenv("MACHINE_BINARY"); machineBin != "" { - machineBinary = machineBin - } else { - whichCmd := exec.Command("which", "machine") - out, _, err := runCommandWithOutput(whichCmd) - if err == nil { - machineBinary = stripTrailingCharacters(out) - } else { - fmt.Printf("ERROR: couldn't resolve full path to the Machine binary") - os.Exit(1) - } - } -} diff --git a/_integration-test/utils.go b/_integration-test/utils.go deleted file mode 100644 index d4004e9ad3..0000000000 --- a/_integration-test/utils.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "fmt" - "os/exec" - "strings" - "syscall" -) - -func getExitCode(err error) (int, error) { - exitCode := 0 - if exiterr, ok := err.(*exec.ExitError); ok { - if procExit := exiterr.Sys().(syscall.WaitStatus); ok { - return procExit.ExitStatus(), nil - } - } - return exitCode, fmt.Errorf("failed to get exit code") -} - -func processExitCode(err error) (exitCode int) { - if err != nil { - var exiterr error - if exitCode, exiterr = getExitCode(err); exiterr != nil { - // TODO: Fix this so we check the error's text. - // we've failed to retrieve exit code, so we set it to 127 - exitCode = 127 - } - } - return -} - -func runCommandWithOutput(cmd *exec.Cmd) (output string, exitCode int, err error) { - exitCode = 0 - out, err := cmd.CombinedOutput() - exitCode = processExitCode(err) - output = string(out) - return - -} - -func stripTrailingCharacters(target string) string { - target = strings.Trim(target, "\n") - target = strings.Trim(target, " ") - return target -} diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000000..9bb348ea1a --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +env-* diff --git a/test/cli.bats b/test/cli.bats new file mode 100644 index 0000000000..75cc8653d7 --- /dev/null +++ b/test/cli.bats @@ -0,0 +1,106 @@ +#!/usr/bin/env bats + +load vars + +@test "cli: show info" { + run ./docker-machine_$PLATFORM-$ARCH + [ "$status" -eq 0 ] + [[ ${lines[0]} =~ "NAME:" ]] + [[ ${lines[1]} =~ "Create and manage machines running Docker" ]] +} + +@test "cli: show active help" { + run ./docker-machine_$PLATFORM-$ARCH active -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command active" ]] +} + +@test "cli: show config help" { + run ./docker-machine_$PLATFORM-$ARCH config -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command config" ]] +} + +@test "cli: show inspect help" { + run ./docker-machine_$PLATFORM-$ARCH inspect -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command inspect" ]] +} + +@test "cli: show ip help" { + run ./docker-machine_$PLATFORM-$ARCH ip -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command ip" ]] +} + +@test "cli: show kill help" { + run ./docker-machine_$PLATFORM-$ARCH kill -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command kill" ]] +} + +@test "cli: show ls help" { + run ./docker-machine_$PLATFORM-$ARCH ls -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command ls" ]] +} + +@test "cli: show restart help" { + run ./docker-machine_$PLATFORM-$ARCH restart -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command restart" ]] +} + +@test "cli: show rm help" { + run ./docker-machine_$PLATFORM-$ARCH rm -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command rm" ]] +} + +@test "cli: show env help" { + run ./docker-machine_$PLATFORM-$ARCH env -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command env" ]] +} + +@test "cli: show ssh help" { + run ./docker-machine_$PLATFORM-$ARCH ssh -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command ssh" ]] +} + +@test "cli: show start help" { + run ./docker-machine_$PLATFORM-$ARCH start -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command start" ]] +} + +@test "cli: show stop help" { + run ./docker-machine_$PLATFORM-$ARCH stop -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command stop" ]] +} + +@test "cli: show upgrade help" { + run ./docker-machine_$PLATFORM-$ARCH upgrade -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command upgrade" ]] +} + +@test "cli: show url help" { + run ./docker-machine_$PLATFORM-$ARCH url -h + [ "$status" -eq 0 ] + [[ ${lines[3]} =~ "command url" ]] +} + +@test "flag: show version" { + run ./docker-machine_$PLATFORM-$ARCH -v + [ "$status" -eq 0 ] + [[ ${lines[0]} =~ "version" ]] +} + +@test "flag: show help" { + run ./docker-machine_$PLATFORM-$ARCH --help + [ "$status" -eq 0 ] + [[ ${lines[0]} =~ "NAME" ]] +} diff --git a/test/driver.bats b/test/driver.bats new file mode 100644 index 0000000000..5001e9738f --- /dev/null +++ b/test/driver.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats + +load vars + +if [ -z "$DRIVER" ]; then + echo "You must set the DRIVER environment variable" + exit 1 +fi + +NAME="bats-$DRIVER-test" +MACHINE_STORAGE_PATH=/tmp/machine-bats-test-$DRIVER + +@test "$DRIVER: machine should not exist" { + run ./docker-machine_$PLATFORM-$ARCH active $NAME + [ "$status" -eq 1 ] +} + +@test "$DRIVER: create" { + run ./docker-machine_$PLATFORM-$ARCH create -d $DRIVER $NAME + [ "$status" -eq 0 ] +} + +@test "$DRIVER: active" { + run ./docker-machine_$PLATFORM-$ARCH active $NAME + [ "$status" -eq 0 ] +} + +@test "$DRIVER: ls" { + run ./docker-machine_$PLATFORM-$ARCH ls + [ "$status" -eq 0 ] + [[ ${lines[1]} =~ "$NAME" ]] + [[ ${lines[1]} =~ "*" ]] +} + +@test "$DRIVER: url" { + run ./docker-machine_$PLATFORM-$ARCH url $NAME + [ "$status" -eq 0 ] +} + +@test "$DRIVER: ip" { + run ./docker-machine_$PLATFORM-$ARCH ip $NAME + [ "$status" -eq 0 ] +} + +@test "$DRIVER: ssh" { + run ./docker-machine_$PLATFORM-$ARCH ssh $NAME -- ls -lah / + [ "$status" -eq 0 ] + [[ ${lines[0]} =~ "total" ]] +} + +case "$DRIVER" in + [digitalocean,amazonec2,virtualbox]*) + @test "$DRIVER: stop" { + run ./docker-machine_$PLATFORM-$ARCH stop $NAME + [ "$status" -eq 0 ] + } + + @test "$DRIVER: machine should show stopped" { + run ./docker-machine_$PLATFORM-$ARCH ls + [ "$status" -eq 0 ] + [[ ${lines[1]} =~ "$NAME" ]] + [[ ${lines[1]} =~ "Stopped" ]] + } + + @test "$DRIVER: start" { + run ./docker-machine_$PLATFORM-$ARCH start $NAME + [ "$status" -eq 0 ] + } + + @test "$DRIVER: machine should show running after start" { + run ./docker-machine_$PLATFORM-$ARCH ls + [ "$status" -eq 0 ] + [[ ${lines[1]} =~ "$NAME" ]] + [[ ${lines[1]} =~ "Running" ]] + } + + @test "$DRIVER: restart" { + run ./docker-machine_$PLATFORM-$ARCH restart $NAME + [ "$status" -eq 0 ] + } + + @test "$DRIVER: machine should show running after restart" { + run ./docker-machine_$PLATFORM-$ARCH ls + [ "$status" -eq 0 ] + [[ ${lines[1]} =~ "$NAME" ]] + [[ ${lines[1]} =~ "Running" ]] + } + + ;; +esac + +@test "$DRIVER: remove" { + run ./docker-machine_$PLATFORM-$ARCH rm $NAME + [ "$status" -eq 0 ] +} + +@test "$DRIVER: machine should not exist" { + run ./docker-machine_$PLATFORM-$ARCH active $NAME + [ "$status" -eq 1 ] +} + diff --git a/test/vars.bash b/test/vars.bash new file mode 100644 index 0000000000..fe3a5fb346 --- /dev/null +++ b/test/vars.bash @@ -0,0 +1,12 @@ +#!/bin/bash + +PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` +ARCH=`uname -m` + +if [ "$ARCH" = "x86_64" ]; then + ARCH="amd64" +else + ARCH="386" +fi + +DRIVER=${DRIVER:-}