Implement docker-machine scp

Signed-off-by: Nathan LeClaire <nathan.leclaire@gmail.com>

commands/commands.go

@@ -339,6 +339,18 @@ var Commands = []cli.Command{
 		Description: "Arguments are [machine-name] [command]",
 		Action:      cmdSsh,
 	},
+	{
+		Name:        "scp",
+		Usage:       "Copy files between machines",
+		Description: "Arguments are [machine:][path] [machine:][path].",
+		Action:      cmdScp,
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name:  "recursive, r",
+				Usage: "Copy files recursively (required to copy directories)",
+			},
+		},
+	},
 	{
 		Name:        "start",
 		Usage:       "Start a machine",

commands/scp.go

@@ -0,0 +1,141 @@
+package commands
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/codegangsta/cli"
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/log"
+)
+
+var (
+	ErrMalformedInput = fmt.Errorf("The input was malformed")
+)
+
+var (
+	// TODO: possibly move this to ssh package
+	baseSSHArgs = []string{
+		"-o", "IdentitiesOnly=yes",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "LogLevel=quiet", // suppress "Warning: Permanently added '[localhost]:2022' (ECDSA) to the list of known hosts."
+	}
+)
+
+func getInfoForScpArg(hostAndPath string, mcn libmachine.Machine) (*libmachine.Host, string, []string, error) {
+	// TODO: What to do about colon in filepath?
+	splitInfo := strings.Split(hostAndPath, ":")
+
+	// Host path.  e.g. "/tmp/foo"
+	if len(splitInfo) == 1 {
+		return nil, splitInfo[0], nil, nil
+	}
+
+	// Remote path.  e.g. "machinename:/usr/bin/cmatrix"
+	if len(splitInfo) == 2 {
+		path := splitInfo[1]
+		host, err := mcn.Get(splitInfo[0])
+		if err != nil {
+			return nil, "", nil, fmt.Errorf("Error loading host: %s", err)
+		}
+		args := []string{
+			"-i",
+			host.Driver.GetSSHKeyPath(),
+		}
+		return host, path, args, nil
+	}
+
+	return nil, "", nil, ErrMalformedInput
+}
+
+func generateLocationArg(host *libmachine.Host, path string) (string, error) {
+	locationPrefix := ""
+	if host != nil {
+		ip, err := host.Driver.GetIP()
+		if err != nil {
+			return "", err
+		}
+		locationPrefix = fmt.Sprintf("%s@%s:", host.Driver.GetSSHUsername(), ip)
+	}
+	return locationPrefix + path, nil
+}
+
+func getScpCmd(src, dest string, sshArgs []string, mcn libmachine.Machine) (*exec.Cmd, error) {
+	cmdPath, err := exec.LookPath("scp")
+	if err != nil {
+		return nil, errors.New("Error: You must have a copy of the scp binary locally to use the scp feature.")
+	}
+
+	srcHost, srcPath, srcOpts, err := getInfoForScpArg(src, mcn)
+	if err != nil {
+		return nil, err
+	}
+
+	destHost, destPath, destOpts, err := getInfoForScpArg(dest, mcn)
+	if err != nil {
+		return nil, err
+	}
+
+	// Append needed -i / private key flags to command.
+	sshArgs = append(sshArgs, srcOpts...)
+	sshArgs = append(sshArgs, destOpts...)
+
+	// Append actual arguments for the scp command (i.e. docker@<ip>:/path)
+	locationArg, err := generateLocationArg(srcHost, srcPath)
+	if err != nil {
+		return nil, err
+	}
+	sshArgs = append(sshArgs, locationArg)
+	locationArg, err = generateLocationArg(destHost, destPath)
+	if err != nil {
+		return nil, err
+	}
+	sshArgs = append(sshArgs, locationArg)
+
+	cmd := exec.Command(cmdPath, sshArgs...)
+	log.Debug(*cmd)
+	return cmd, nil
+}
+
+func runCmdWithStdIo(cmd exec.Cmd) error {
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func cmdScp(c *cli.Context) {
+	args := c.Args()
+	if len(args) != 2 {
+		cli.ShowCommandHelp(c, "scp")
+		log.Fatal("Improper number of arguments.")
+	}
+
+	// TODO: Check that "-3" flag is available in user's version of scp.
+	// It is on every system I've checked, but the manual mentioned it's "newer"
+	sshArgs := append(baseSSHArgs, "-3")
+
+	if c.Bool("recursive") {
+		sshArgs = append(sshArgs, "-r")
+	}
+
+	src := args[0]
+	dest := args[1]
+
+	mcn := getDefaultMcn(c)
+	cmd, err := getScpCmd(src, dest, sshArgs, *mcn)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := runCmdWithStdIo(*cmd); err != nil {
+		log.Fatal(err)
+	}
+}

commands/scp_test.go

@@ -0,0 +1,258 @@
+package commands
+
+import (
+	"errors"
+	"fmt"
+	"os/exec"
+	"reflect"
+	"testing"
+
+	"github.com/docker/machine/drivers"
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/provider"
+	"github.com/docker/machine/state"
+)
+
+type ScpFakeDriver struct {
+	MockState state.State
+}
+
+type ScpFakeStore struct{}
+
+func (d ScpFakeDriver) AuthorizePort(ports []*drivers.Port) error {
+	return nil
+}
+
+func (d ScpFakeDriver) DeauthorizePort(ports []*drivers.Port) error {
+	return nil
+}
+
+func (d ScpFakeDriver) DriverName() string {
+	return "fake"
+}
+
+func (d ScpFakeDriver) SetConfigFromFlags(flags drivers.DriverOptions) error {
+	return nil
+}
+
+func (d ScpFakeDriver) GetURL() (string, error) {
+	return "", nil
+}
+
+func (d ScpFakeDriver) GetIP() (string, error) {
+	return "12.34.56.78", nil
+}
+
+func (d ScpFakeDriver) GetState() (state.State, error) {
+	return d.MockState, nil
+}
+
+func (d ScpFakeDriver) GetMachineName() string {
+	return "myfunhost"
+}
+
+func (d ScpFakeDriver) GetSSHHostname() (string, error) {
+	return "12.34.56.76", nil
+}
+
+func (d ScpFakeDriver) GetSSHPort() (int, error) {
+	return 22, nil
+}
+
+func (d ScpFakeDriver) PreCreateCheck() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Create() error {
+	return nil
+}
+
+func (d ScpFakeDriver) GetProviderType() provider.ProviderType {
+	return provider.Local
+}
+
+func (d ScpFakeDriver) Remove() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Start() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Stop() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Restart() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Kill() error {
+	return nil
+}
+
+func (d ScpFakeDriver) Upgrade() error {
+	return nil
+}
+
+func (d ScpFakeDriver) StartDocker() error {
+	return nil
+}
+
+func (d ScpFakeDriver) StopDocker() error {
+	return nil
+}
+
+func (d ScpFakeDriver) GetDockerConfigDir() string {
+	return ""
+}
+
+func (d ScpFakeDriver) GetSSHCommand(args ...string) (*exec.Cmd, error) {
+	return &exec.Cmd{}, nil
+}
+
+func (d ScpFakeDriver) GetSSHUsername() string {
+	return "root"
+}
+
+func (d ScpFakeDriver) GetSSHKeyPath() string {
+	return "/fake/keypath/id_rsa"
+}
+
+func (s ScpFakeStore) Exists(name string) (bool, error) {
+	return true, nil
+}
+
+func (s ScpFakeStore) GetActive() (*libmachine.Host, error) {
+	return nil, nil
+}
+
+func (s ScpFakeStore) GetPath() string {
+	return ""
+}
+
+func (s ScpFakeStore) GetCACertificatePath() (string, error) {
+	return "", nil
+}
+
+func (s ScpFakeStore) GetPrivateKeyPath() (string, error) {
+	return "", nil
+}
+
+func (s ScpFakeStore) List() ([]*libmachine.Host, error) {
+	return nil, nil
+}
+
+func (s ScpFakeStore) Get(name string) (*libmachine.Host, error) {
+	if name == "myfunhost" {
+		return &libmachine.Host{
+			Name:   "myfunhost",
+			Driver: ScpFakeDriver{},
+		}, nil
+	}
+	return nil, errors.New("Host not found")
+}
+
+func (s ScpFakeStore) Remove(name string, force bool) error {
+	return nil
+}
+
+func (s ScpFakeStore) Save(host *libmachine.Host) error {
+	return nil
+}
+
+func TestGetInfoForScpArg(t *testing.T) {
+	mcn, _ := libmachine.New(ScpFakeStore{})
+
+	expectedPath := "/tmp/foo"
+	host, path, opts, err := getInfoForScpArg("/tmp/foo", *mcn)
+	if err != nil {
+		t.Fatalf("Unexpected error in local getInfoForScpArg call: %s", err)
+	}
+	if path != expectedPath {
+		t.Fatalf("Path %s not equal to expected path %s", path, expectedPath)
+	}
+	if host != nil {
+		t.Fatal("host should be nil")
+	}
+	if opts != nil {
+		t.Fatal("opts should be nil")
+	}
+
+	host, path, opts, err = getInfoForScpArg("myfunhost:/home/docker/foo", *mcn)
+	if err != nil {
+		t.Fatal("Unexpected error in machine-based getInfoForScpArg call: %s", err)
+	}
+	expectedOpts := []string{
+		"-i",
+		"/fake/keypath/id_rsa",
+	}
+	for i := range opts {
+		if expectedOpts[i] != opts[i] {
+			t.Fatalf("Mismatch in returned opts: %s != %s", expectedOpts[i], opts[i])
+		}
+	}
+	if host.Name != "myfunhost" {
+		t.Fatal("Expected host.Name to be myfunhost, got %s", host.Name)
+	}
+	if path != "/home/docker/foo" {
+		t.Fatalf("Expected path to be /home/docker/foo, got %s", path)
+	}
+
+	host, path, opts, err = getInfoForScpArg("foo:bar:widget", *mcn)
+	if err != ErrMalformedInput {
+		t.Fatalf("Didn't get back an error when we were expecting it for malformed args")
+	}
+}
+
+func TestGenerateLocationArg(t *testing.T) {
+	host := libmachine.Host{
+		Driver: ScpFakeDriver{},
+	}
+
+	// local arg
+	arg, err := generateLocationArg(nil, "/home/docker/foo")
+	if err != nil {
+		t.Fatalf("Unexpected error generating location arg for local: %s", err)
+	}
+	if arg != "/home/docker/foo" {
+		t.Fatalf("Expected arg to be /home/docker/foo, was %s", arg)
+	}
+
+	arg, err = generateLocationArg(&host, "/home/docker/foo")
+	if err != nil {
+		t.Fatalf("Unexpected error generating location arg for remote: %s", err)
+	}
+	if arg != "root@12.34.56.78:/home/docker/foo" {
+		t.Fatalf("Expected arg to be root@12.34.56.78, instead it was %s", arg)
+	}
+}
+
+func TestGetScpCmd(t *testing.T) {
+	mcn, _ := libmachine.New(ScpFakeStore{})
+
+	// TODO: This is a little "integration-ey".  Perhaps
+	// make an ScpDispatcher (name?) interface so that the reliant
+	// methods can be mocked.
+	expectedArgs := append(
+		baseSSHArgs,
+		"-3",
+		"-i",
+		"/fake/keypath/id_rsa",
+		"/tmp/foo",
+		"root@12.34.56.78:/home/docker/foo",
+	)
+	expectedCmd := exec.Command("/usr/bin/scp", expectedArgs...)
+
+	cmd, err := getScpCmd("/tmp/foo", "myfunhost:/home/docker/foo", append(baseSSHArgs, "-3"), *mcn)
+	if err != nil {
+		t.Fatalf("Unexpected err getting scp command: %s", err)
+	}
+
+	correct := reflect.DeepEqual(expectedCmd, cmd)
+	if !correct {
+		fmt.Println(expectedCmd)
+		fmt.Println(cmd)
+		t.Fatal("Expected scp cmd structs to be equal but there was mismatch")
+	}
+}

docs/index.md

@@ -913,6 +913,33 @@ cgroup                  499.8M         0    499.8M   0% /sys/fs/cgroup
 /mnt/sda1/var/lib/docker/aufs
 ```
 
+#### scp
+
+Copy files from your local host to a machine, from machine to machine, or from a
+machine to your local host using `scp`.
+
+The notation is `machinename:/path/to/files` for the arguments; in the host
+machine's case, you don't have to specify the name, just the path.
+
+Consider the following example:
+
+```
+$ cat foo.txt
+cat: foo.txt: No such file or directory
+$ docker-machine ssh dev pwd
+/home/docker
+$ docker-machine ssh dev 'echo A file created remotely! >foo.txt'
+$ docker-machine scp dev:/home/docker/foo.txt .
+foo.txt                                                           100%   28     0.0KB/s   00:00
+$ cat foo.txt
+A file created remotely!
+```
+
+Files are copied recursively by default (`scp`'s `-r` flag).
+
+In the case of transfering files from machine to machine, they go through the
+local host's filesystem first (using `scp`'s `-3` flag).
+
 #### start
 
 Gracefully start a machine.

test/integration/scp.bats

@@ -0,0 +1,41 @@
+#!/usr/bin/env bats
+
+load helpers
+
+export DRIVER=virtualbox
+export NAME="bats-$DRIVER-test"
+export MACHINE_STORAGE_PATH=/tmp/machine-bats-test-$DRIVER
+export SECOND_MACHINE="$NAME-2"
+
+@test "$DRIVER: create" {
+  run machine create -d $DRIVER $NAME
+  [[ ${status} -eq 0 ]]
+}
+
+@test "$DRIVER: test machine scp command from remote to host" {
+  machine ssh $NAME 'echo A file created remotely! >/tmp/foo.txt'
+  machine scp $NAME:/tmp/foo.txt .
+  [[ $(cat foo.txt) == "A file created remotely!" ]]
+}
+
+@test "$DRIVER: test machine scp command from host to remote" {
+  echo A file created locally! >foo.txt
+  machine scp foo.txt $NAME:/tmp/foo.txt
+  [[ $(machine ssh $NAME cat /tmp/foo.txt) == "A file created locally!" ]]
+}
+
+@test "$DRIVER: create machine to test transferring files from machine to machine" {
+  run machine create -d $DRIVER $SECOND_MACHINE
+  [[ ${status} -eq 0 ]]
+}
+
+@test "$DRIVER: scp from one machine to another" {
+  run machine ssh $NAME 'echo A file hopping around! >/tmp/foo.txt'
+  run machine scp $NAME:/tmp/foo.txt $SECOND_MACHINE:/tmp/foo.txt
+  [[ $(machine ssh ${SECOND_MACHINE} cat /tmp/foo.txt) == "A file hopping around!" ]]
+}
+
+@test "cleanup" {
+  rm foo.txt
+  machine rm $NAME
+}