From 47bd2622609647004782aa4bf4b5c6d74accd59a Mon Sep 17 00:00:00 2001 From: Nathan LeClaire Date: Wed, 6 May 2015 17:51:35 -0700 Subject: [PATCH] Implement configurable Swarm options Signed-off-by: Nathan LeClaire --- commands/commands.go | 10 ++ commands/create.go | 12 ++- docs/index.md | 34 +++++++ libmachine/provision/boot2docker.go | 2 +- libmachine/provision/configure_swarm.go | 120 ++++++++++++++++++++++++ libmachine/provision/rancheros.go | 2 +- libmachine/provision/redhat.go | 2 +- libmachine/provision/ubuntu.go | 2 +- libmachine/provision/utils.go | 56 ----------- libmachine/swarm/swarm.go | 25 ++--- test/integration/swarm-options.bats | 38 ++++++++ 11 files changed, 226 insertions(+), 77 deletions(-) create mode 100644 libmachine/provision/configure_swarm.go create mode 100644 test/integration/swarm-options.bats diff --git a/commands/commands.go b/commands/commands.go index 4256048c28..9749f29a62 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -203,6 +203,16 @@ var sharedCreateFlags = []cli.Flag{ Usage: "Discovery service to use with Swarm", Value: "", }, + cli.StringFlag{ + Name: "swarm-strategy", + Usage: "Define a default scheduling strategy for Swarm", + Value: "spread", + }, + cli.StringSliceFlag{ + Name: "swarm-opt", + Usage: "Define arbitrary flags for swarm", + Value: &cli.StringSlice{}, + }, cli.StringFlag{ Name: "swarm-host", Usage: "ip/socket to listen on for Swarm master", diff --git a/commands/create.go b/commands/create.go index 3a2302c712..bb3cb41bfa 100644 --- a/commands/create.go +++ b/commands/create.go @@ -77,11 +77,13 @@ func cmdCreate(c *cli.Context) { TlsVerify: true, }, SwarmOptions: &swarm.SwarmOptions{ - IsSwarm: c.Bool("swarm"), - Master: c.Bool("swarm-master"), - Discovery: c.String("swarm-discovery"), - Address: c.String("swarm-addr"), - Host: c.String("swarm-host"), + IsSwarm: c.Bool("swarm"), + Master: c.Bool("swarm-master"), + Discovery: c.String("swarm-discovery"), + Address: c.String("swarm-addr"), + Host: c.String("swarm-host"), + Strategy: c.String("swarm-strategy"), + ArbitraryFlags: c.StringSlice("swarm-opt"), }, } diff --git a/docs/index.md b/docs/index.md index e536e042b8..80f0e28d72 100644 --- a/docs/index.md +++ b/docs/index.md @@ -592,6 +592,40 @@ $ docker-machine create -d virtualbox \ gdns ``` +##### Specifying Swarm options for the created machine + +In addition to being able to configure Docker Engine options as listed above, +you can use Machine to specify how the created Swarm master should be +configured). There is a `--swarm-strategy` flag, which you can use to specify +the [scheduling strategy](https://docs.docker.com/swarm/scheduler/strategy/) +which Docker Swarm should use (Machine defaults to the `spread` strategy). +There is also a general purpose `--swarm-opt` option which works similar to how +the aforementioned `--engine-opt` option does, except that it specifies options +for the `swarm manage` command (used to boot a master node) instead of the base +command. You can use this to configure features that power users might be +interested in, such as configuring the heartbeat interval or Swarm's willingness +to over-commit resources. + +If you're not sure how to configure these options, it is best to not specify +configuration at all. Docker Machine will choose sensible defaults for you and +you won't have to worry about it. + +Example create: + +``` +$ docker-machine create -d virtualbox \ + --swarm \ + --swarm-master \ + --swarm-discovery token:// \ + --swarm-strategy binpack \ + --swarm-opt heartbeat=5 \ + upbeat +``` + +This will set the swarm scheduling strategy to "binpack" (pack in containers as +tightly as possible per host instead of spreading them out), and the "heartbeat" +interval to 5 seconds. + #### config Show the Docker client configuration for a machine. diff --git a/libmachine/provision/boot2docker.go b/libmachine/provision/boot2docker.go index c8619c458c..e743ca129f 100644 --- a/libmachine/provision/boot2docker.go +++ b/libmachine/provision/boot2docker.go @@ -213,7 +213,7 @@ func (provisioner *Boot2DockerProvisioner) Provision(swarmOptions swarm.SwarmOpt return err } - if err := configureSwarm(provisioner, swarmOptions); err != nil { + if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil { return err } diff --git a/libmachine/provision/configure_swarm.go b/libmachine/provision/configure_swarm.go new file mode 100644 index 0000000000..85ebb2ce35 --- /dev/null +++ b/libmachine/provision/configure_swarm.go @@ -0,0 +1,120 @@ +package provision + +import ( + "bytes" + "fmt" + "net/url" + "strings" + "text/template" + + "github.com/docker/machine/libmachine/auth" + "github.com/docker/machine/libmachine/swarm" + "github.com/docker/machine/log" +) + +type SwarmCommandContext struct { + ContainerName string + DockerDir string + DockerPort int + Ip string + Port string + AuthOptions auth.AuthOptions + SwarmOptions swarm.SwarmOptions + SwarmImage string +} + +// Wrapper function to generate a docker run swarm command (manage or join) +// from a template/context and execute it. +func runSwarmCommandFromTemplate(p Provisioner, cmdTmpl string, swarmCmdContext SwarmCommandContext) error { + var ( + executedCmdTmpl bytes.Buffer + ) + + parsedMasterCmdTemplate, err := template.New("swarmMasterCmd").Parse(cmdTmpl) + if err != nil { + return err + } + + parsedMasterCmdTemplate.Execute(&executedCmdTmpl, swarmCmdContext) + + log.Debugf("The swarm command being run is: %s", executedCmdTmpl.String()) + + if _, err := p.SSHCommand(executedCmdTmpl.String()); err != nil { + return err + } + + return nil +} + +func configureSwarm(p Provisioner, swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions) error { + if !swarmOptions.IsSwarm { + return nil + } + + ip, err := p.GetDriver().GetIP() + if err != nil { + return err + } + + u, err := url.Parse(swarmOptions.Host) + if err != nil { + return err + } + + parts := strings.Split(u.Host, ":") + port := parts[1] + + dockerDir := p.GetDockerOptionsDir() + + swarmCmdContext := SwarmCommandContext{ + ContainerName: "", + DockerDir: dockerDir, + DockerPort: 2376, + Ip: ip, + Port: port, + AuthOptions: authOptions, + SwarmOptions: swarmOptions, + SwarmImage: swarm.DockerImage, + } + + // First things first, get the swarm image. + if _, err := p.SSHCommand(fmt.Sprintf("sudo docker pull %s", swarm.DockerImage)); err != nil { + return err + } + + swarmMasterCmdTemplate := `sudo docker run -d \ +--restart=always \ +--name swarm-agent-master \ +-p {{.Port}}:{{.Port}} \ +-v {{.DockerDir}}:{{.DockerDir}} \ +{{.SwarmImage}} \ +manage \ +--tlsverify \ +--tlscacert={{.AuthOptions.CaCertRemotePath}} \ +--tlscert={{.AuthOptions.ServerCertRemotePath}} \ +--tlskey={{.AuthOptions.ServerKeyRemotePath}} \ +-H {{.SwarmOptions.Host}} \ +--strategy {{.SwarmOptions.Strategy}} {{range .SwarmOptions.ArbitraryFlags}} --{{.}}{{end}} {{.SwarmOptions.Discovery}} +` + + swarmWorkerCmdTemplate := `sudo docker run -d \ +--restart=always \ +--name swarm-agent \ +{{.SwarmImage}} \ +join --addr {{.Ip}}:{{.DockerPort}} {{.SwarmOptions.Discovery}} +` + + if swarmOptions.Master { + log.Debug("Launching swarm master") + if err := runSwarmCommandFromTemplate(p, swarmMasterCmdTemplate, swarmCmdContext); err != nil { + return err + } + } + + log.Debug("Launch swarm worker") + if err := runSwarmCommandFromTemplate(p, swarmWorkerCmdTemplate, swarmCmdContext); err != nil { + return err + } + + return nil +} diff --git a/libmachine/provision/rancheros.go b/libmachine/provision/rancheros.go index b82e7070ac..c71a2adebe 100644 --- a/libmachine/provision/rancheros.go +++ b/libmachine/provision/rancheros.go @@ -117,7 +117,7 @@ func (provisioner *RancherProvisioner) Provision(swarmOptions swarm.SwarmOptions } log.Debugf("Configuring swarm") - if err := configureSwarm(provisioner, swarmOptions); err != nil { + if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil { return err } diff --git a/libmachine/provision/redhat.go b/libmachine/provision/redhat.go index fdfaa13b37..4dec0d955c 100644 --- a/libmachine/provision/redhat.go +++ b/libmachine/provision/redhat.go @@ -174,7 +174,7 @@ func (provisioner *RedHatProvisioner) Provision(swarmOptions swarm.SwarmOptions, return err } - if err := configureSwarm(provisioner, swarmOptions); err != nil { + if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil { return err } diff --git a/libmachine/provision/ubuntu.go b/libmachine/provision/ubuntu.go index 27e2632387..16cbb58665 100644 --- a/libmachine/provision/ubuntu.go +++ b/libmachine/provision/ubuntu.go @@ -131,7 +131,7 @@ func (provisioner *UbuntuProvisioner) Provision(swarmOptions swarm.SwarmOptions, return err } - if err := configureSwarm(provisioner, swarmOptions); err != nil { + if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil { return err } diff --git a/libmachine/provision/utils.go b/libmachine/provision/utils.go index 71911a9f81..d3b9d6db0e 100644 --- a/libmachine/provision/utils.go +++ b/libmachine/provision/utils.go @@ -11,7 +11,6 @@ import ( "github.com/docker/machine/libmachine/auth" "github.com/docker/machine/libmachine/provision/pkgaction" - "github.com/docker/machine/libmachine/swarm" "github.com/docker/machine/log" "github.com/docker/machine/utils" ) @@ -180,58 +179,3 @@ func ConfigureAuth(p Provisioner) error { return nil } - -func configureSwarm(p Provisioner, swarmOptions swarm.SwarmOptions) error { - if !swarmOptions.IsSwarm { - return nil - } - - log.Debug("configuring swarm") - - basePath := p.GetDockerOptionsDir() - ip, err := p.GetDriver().GetIP() - if err != nil { - return err - } - - tlsCaCert := path.Join(basePath, "ca.pem") - tlsCert := path.Join(basePath, "server.pem") - tlsKey := path.Join(basePath, "server-key.pem") - masterArgs := fmt.Sprintf("--tlsverify --tlscacert=%s --tlscert=%s --tlskey=%s -H %s %s", - tlsCaCert, tlsCert, tlsKey, swarmOptions.Host, swarmOptions.Discovery) - nodeArgs := fmt.Sprintf("--addr %s:2376 %s", ip, swarmOptions.Discovery) - - u, err := url.Parse(swarmOptions.Host) - if err != nil { - return err - } - - parts := strings.Split(u.Host, ":") - port := parts[1] - - if _, err := p.SSHCommand(fmt.Sprintf("sudo docker pull %s", swarm.DockerImage)); err != nil { - return err - } - - dockerDir := p.GetDockerOptionsDir() - - // if master start master agent - if swarmOptions.Master { - log.Debug("launching swarm master") - log.Debugf("master args: %s", masterArgs) - if _, err = p.SSHCommand(fmt.Sprintf("sudo docker run -d -p %s:%s --restart=always --name swarm-agent-master -v %s:%s %s manage %s", - port, port, dockerDir, dockerDir, swarm.DockerImage, masterArgs)); err != nil { - return err - } - } - - // start node agent - log.Debug("launching swarm node") - log.Debugf("node args: %s", nodeArgs) - if _, err = p.SSHCommand(fmt.Sprintf("sudo docker run -d --restart=always --name swarm-agent -v %s:%s %s join %s", - dockerDir, dockerDir, swarm.DockerImage, nodeArgs)); err != nil { - return err - } - - return nil -} diff --git a/libmachine/swarm/swarm.go b/libmachine/swarm/swarm.go index 2c752c5dd7..297194a61e 100644 --- a/libmachine/swarm/swarm.go +++ b/libmachine/swarm/swarm.go @@ -6,16 +6,17 @@ const ( ) type SwarmOptions struct { - IsSwarm bool - Address string - Discovery string - Master bool - Host string - Strategy string - Heartbeat int - Overcommit float64 - TlsCaCert string - TlsCert string - TlsKey string - TlsVerify bool + IsSwarm bool + Address string + Discovery string + Master bool + Host string + Strategy string + Heartbeat int + Overcommit float64 + TlsCaCert string + TlsCert string + TlsKey string + TlsVerify bool + ArbitraryFlags []string } diff --git a/test/integration/swarm-options.bats b/test/integration/swarm-options.bats new file mode 100644 index 0000000000..36686f5362 --- /dev/null +++ b/test/integration/swarm-options.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats + +load helpers + +export DRIVER=virtualbox +export MACHINE_STORAGE_PATH=/tmp/machine-bats-test-$DRIVER +export TOKEN=$(curl -sS -X POST "https://discovery-stage.hub.docker.com/v1/clusters") + +@test "create swarm master" { + run machine create -d virtualbox --swarm --swarm-master --swarm-discovery "token://$TOKEN" --swarm-strategy binpack --swarm-opt heartbeat=5 queenbee + [[ "$status" -eq 0 ]] +} + +@test "create swarm node" { + run machine create -d virtualbox --swarm --swarm-discovery "token://$TOKEN" workerbee + [[ "$status" -eq 0 ]] +} + +@test "ensure strategy is correct" { + strategy=$(docker $(machine config --swarm queenbee) info | grep "Strategy:" | awk '{ print $2 }') + echo ${strategy} + [[ "$strategy" == "binpack" ]] +} + +@test "ensure heartbeat" { + heartbeat_arg=$(docker $(machine config queenbee) inspect -f '{{index .Args 9}}' swarm-agent-master) + echo ${heartbeat_arg} + [[ "$heartbeat_arg" == "--heartbeat=5" ]] +} + +@test "clean up created nodes" { + run machine rm queenbee workerbee + [[ "$status" -eq 0 ]] +} + +@test "remove dir" { + rm -rf $MACHINE_STORAGE_PATH +}