From 0d2bc08d6ee69924dbee098c94a87da40634fb1f Mon Sep 17 00:00:00 2001 From: Jean-Laurent de Morlhon Date: Wed, 13 Jan 2016 18:51:45 +0100 Subject: [PATCH 1/5] Adding ec2client interface to allow unit testing. Signed-off-by: Jean-Laurent de Morlhon --- drivers/amazonec2/amazonec2.go | 38 ++++++++++++++------- drivers/amazonec2/amazonec2_test.go | 5 +-- drivers/amazonec2/ec2client.go | 51 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 14 deletions(-) create mode 100644 drivers/amazonec2/ec2client.go diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go index 9a2e0af4d1..6033f486bb 100644 --- a/drivers/amazonec2/amazonec2.go +++ b/drivers/amazonec2/amazonec2.go @@ -3,6 +3,7 @@ package amazonec2 import ( "crypto/md5" "crypto/rand" + "errors" "fmt" "io" "io/ioutil" @@ -52,6 +53,7 @@ var ( type Driver struct { *drivers.BaseDriver + clientFactory func() Ec2Client Id string AccessKey string SecretKey string @@ -83,6 +85,10 @@ type Driver struct { Monitoring bool } +type clientFactory interface { + build(d *Driver) Ec2Client +} + func (d *Driver) GetCreateFlags() []mcnflag.Flag { return []mcnflag.Flag{ mcnflag.StringFlag{ @@ -201,9 +207,9 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag { } } -func NewDriver(hostName, storePath string) drivers.Driver { +func NewDriver(hostName, storePath string) *Driver { id := generateId() - return &Driver{ + driver := &Driver{ Id: id, AMI: defaultAmiId, Region: defaultRegion, @@ -218,6 +224,24 @@ func NewDriver(hostName, storePath string) drivers.Driver { StorePath: storePath, }, } + + driver.clientFactory = driver.buildClient + + return driver +} + +func (d *Driver) buildClient() Ec2Client { + config := aws.NewConfig() + alogger := AwsLogger() + config = config.WithRegion(d.Region) + config = config.WithCredentials(credentials.NewStaticCredentials(d.AccessKey, d.SecretKey, d.SessionToken)) + config = config.WithLogger(alogger) + config = config.WithLogLevel(aws.LogDebugWithHTTPBody) + return ec2.New(session.New(config)) +} + +func (d *Driver) getClient() Ec2Client { + return d.clientFactory() } func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { @@ -661,16 +685,6 @@ func (d *Driver) Remove() error { return nil } -func (d *Driver) getClient() *ec2.EC2 { - config := aws.NewConfig() - alogger := AwsLogger() - config = config.WithRegion(d.Region) - config = config.WithCredentials(credentials.NewStaticCredentials(d.AccessKey, d.SecretKey, d.SessionToken)) - config = config.WithLogger(alogger) - config = config.WithLogLevel(aws.LogDebugWithHTTPBody) - return ec2.New(session.New(config)) -} - func (d *Driver) getInstance() (*ec2.Instance, error) { instances, err := d.getClient().DescribeInstances(&ec2.DescribeInstancesInput{ InstanceIds: []*string{&d.InstanceId}, diff --git a/drivers/amazonec2/amazonec2_test.go b/drivers/amazonec2/amazonec2_test.go index 70e3021c26..98170f056e 100644 --- a/drivers/amazonec2/amazonec2_test.go +++ b/drivers/amazonec2/amazonec2_test.go @@ -5,6 +5,8 @@ import ( "os" "testing" + "errors" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/docker/machine/commands/commandstest" @@ -85,8 +87,7 @@ func getTestDriver() (*Driver, error) { d := NewDriver(machineTestName, storePath) d.SetConfigFromFlags(getDefaultTestDriverFlags()) - drv := d.(*Driver) - return drv, nil + return d, nil } func TestConfigureSecurityGroupPermissionsEmpty(t *testing.T) { diff --git a/drivers/amazonec2/ec2client.go b/drivers/amazonec2/ec2client.go new file mode 100644 index 0000000000..ecb2a50b19 --- /dev/null +++ b/drivers/amazonec2/ec2client.go @@ -0,0 +1,51 @@ +package amazonec2 + +import "github.com/aws/aws-sdk-go/service/ec2" + +type Ec2Client interface { + DescribeAccountAttributes(input *ec2.DescribeAccountAttributesInput) (*ec2.DescribeAccountAttributesOutput, error) + + DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) + + CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) + + //SecurityGroup + + CreateSecurityGroup(input *ec2.CreateSecurityGroupInput) (*ec2.CreateSecurityGroupOutput, error) + + AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput) (*ec2.AuthorizeSecurityGroupIngressOutput, error) + + DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) + + DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput) (*ec2.DeleteSecurityGroupOutput, error) + + //KeyPair + + DeleteKeyPair(input *ec2.DeleteKeyPairInput) (*ec2.DeleteKeyPairOutput, error) + + ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) + + DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) + + //Instances + + DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) + + StartInstances(input *ec2.StartInstancesInput) (*ec2.StartInstancesOutput, error) + + RebootInstances(input *ec2.RebootInstancesInput) (*ec2.RebootInstancesOutput, error) + + StopInstances(input *ec2.StopInstancesInput) (*ec2.StopInstancesOutput, error) + + RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) + + TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) + + //SpotInstances + + RequestSpotInstances(input *ec2.RequestSpotInstancesInput) (*ec2.RequestSpotInstancesOutput, error) + + DescribeSpotInstanceRequests(input *ec2.DescribeSpotInstanceRequestsInput) (*ec2.DescribeSpotInstanceRequestsOutput, error) + + WaitUntilSpotInstanceRequestFulfilled(input *ec2.DescribeSpotInstanceRequestsInput) error +} From 04fd9e7741af51607dad5846a410f6b145c238c6 Mon Sep 17 00:00:00 2001 From: Jean-Laurent de Morlhon Date: Thu, 14 Jan 2016 11:07:28 +0100 Subject: [PATCH 2/5] Fetching the default vpc id from account Signed-off-by: Jean-Laurent de Morlhon --- drivers/amazonec2/amazonec2.go | 24 ++++++++++- drivers/amazonec2/amazonec2_test.go | 66 +++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go index 6033f486bb..33c0b0fffc 100644 --- a/drivers/amazonec2/amazonec2.go +++ b/drivers/amazonec2/amazonec2.go @@ -289,8 +289,15 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { return fmt.Errorf("amazonec2 driver requires the --amazonec2-secret-key option") } + if d.VpcId == "" { + d.VpcId, err = d.getDefaultVPCId() + if err != nil { + log.Errorf("couldn't determine your account Default VPC ID : %q", err) + } + } + if d.SubnetId == "" && d.VpcId == "" { - return fmt.Errorf("amazonec2 driver requires either the --amazonec2-subnet-id or --amazonec2-vpc-id option") + return fmt.Errorf("amazonec2 driver requires either the --amazonec2-subnet-id or --amazonec2-vpc-id option or an AWS Account with a default vpc-id") } if d.SubnetId != "" && d.VpcId != "" { @@ -952,6 +959,21 @@ func (d *Driver) deleteKeyPair() error { return nil } +func (d *Driver) getDefaultVPCId() (string, error) { + output, err := d.getClient().DescribeAccountAttributes(&ec2.DescribeAccountAttributesInput{}) + if err != nil { + return "", err + } + + for _, attribute := range output.AccountAttributes { + if *attribute.AttributeName == "default-vpc" { + return *attribute.AttributeValues[0].AttributeValue, nil + } + } + + return "", errors.New("No default-vpc attribute") +} + func generateId() string { rb := make([]byte, 10) _, err := rand.Read(rb) diff --git a/drivers/amazonec2/amazonec2_test.go b/drivers/amazonec2/amazonec2_test.go index 98170f056e..6e9c0ee850 100644 --- a/drivers/amazonec2/amazonec2_test.go +++ b/drivers/amazonec2/amazonec2_test.go @@ -239,3 +239,69 @@ func TestSetConfigFromFlags(t *testing.T) { assert.NoError(t, err) assert.Empty(t, checkFlags.InvalidFlags) } + +type fakeEC2WithDescribe struct { + *ec2.EC2 + output *ec2.DescribeAccountAttributesOutput + err error +} + +func (f *fakeEC2WithDescribe) DescribeAccountAttributes(input *ec2.DescribeAccountAttributesInput) (*ec2.DescribeAccountAttributesOutput, error) { + return f.output, f.err +} + +func TestFindDefaultVPC(t *testing.T) { + defaultVpc := "default-vpc" + vpcName := "vpc-9999" + + driver := NewDriver("machineFoo", "path") + driver.clientFactory = func() Ec2Client { + return &fakeEC2WithDescribe{ + output: &ec2.DescribeAccountAttributesOutput{ + AccountAttributes: []*ec2.AccountAttribute{ + { + AttributeName: &defaultVpc, + AttributeValues: []*ec2.AccountAttributeValue{ + {AttributeValue: &vpcName}, + }, + }, + }, + }, + } + } + + vpc, err := driver.getDefaultVPCId() + + assert.Equal(t, "vpc-9999", vpc) + assert.NoError(t, err) +} + +func TestDefaultVPCIsMissing(t *testing.T) { + driver := NewDriver("machineFoo", "path") + driver.clientFactory = func() Ec2Client { + return &fakeEC2WithDescribe{ + output: &ec2.DescribeAccountAttributesOutput{ + AccountAttributes: []*ec2.AccountAttribute{}, + }, + } + } + + vpc, err := driver.getDefaultVPCId() + + assert.EqualError(t, err, "No default-vpc attribute") + assert.Empty(t, vpc) +} + +func TestDescribeAccountAttributeFails(t *testing.T) { + driver := NewDriver("machineFoo", "path") + driver.clientFactory = func() Ec2Client { + return &fakeEC2WithDescribe{ + err: errors.New("Not Found"), + } + } + + vpc, err := driver.getDefaultVPCId() + + assert.EqualError(t, err, "Not Found") + assert.Empty(t, vpc) +} From e298f6849222ee67412b192176acafd32cc1d7ff Mon Sep 17 00:00:00 2001 From: Jean-Laurent de Morlhon Date: Wed, 13 Jan 2016 19:19:44 +0100 Subject: [PATCH 3/5] Documentation update Signed-off-by: Jean-Laurent de Morlhon --- docs/drivers/aws.md | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/drivers/aws.md b/docs/drivers/aws.md index 7f375d66f9..308093d59a 100644 --- a/docs/drivers/aws.md +++ b/docs/drivers/aws.md @@ -10,26 +10,16 @@ parent="smn_machine_drivers" # Amazon Web Services -Create machines on [Amazon Web Services](http://aws.amazon.com). To create machines on [Amazon Web Services](http://aws.amazon.com), you must supply three required parameters: +Create machines on [Amazon Web Services](http://aws.amazon.com). To create machines on [Amazon Web Services](http://aws.amazon.com), you must supply two required parameters: - Access Key ID - Secret Access Key -- VPC ID -Obtain your IDs and Keys from AWS. To find the VPC ID: - -1. Login to the AWS console -2. Go to **Services -> VPC -> Your VPCs**. -3. Locate the VPC ID you want from the _VPC_ column. -4. Go to **Services -> VPC -> Subnets**. Examine the _Availability Zone_ column to verify that zone `a` exists and matches your VPC ID. - - For example, `us-east1-a` is in the `a` availability zone. If the `a` zone is not present, you can create a new subnet in that zone or specify a different zone when you create the machine. +Obtain your IDs and Keys from AWS. To create the machine instance, specify `--driver amazonec2` and the three required parameters. - $ docker-machine create --driver amazonec2 --amazonec2-access-key AKI******* --amazonec2-secret-key 8T93C********* --amazonec2-vpc-id vpc-****** aws01 - -This example assumes the VPC ID was found in the `a` availability zone. Use the`--amazonec2-zone` flag to specify a zone other than the `a` zone. For example, `--amazonec2-zone c` signifies `us-east1-c`. + $ docker-machine create --driver amazonec2 --amazonec2-access-key AKI******* --amazonec2-secret-key 8T93C******* aws01 ## Options @@ -38,7 +28,7 @@ This example assumes the VPC ID was found in the `a` availability zone. Use the` - `--amazonec2-session-token`: Your session token for the Amazon Web Services API. - `--amazonec2-ami`: The AMI ID of the instance to use. - `--amazonec2-region`: The region to use when launching the instance. -- `--amazonec2-vpc-id`: **required** Your VPC ID to launch the instance in. +- `--amazonec2-vpc-id`: Your VPC ID to launch the instance in. - `--amazonec2-zone`: The AWS zone to launch the instance in (i.e. one of a,b,c,d,e). - `--amazonec2-subnet-id`: AWS VPC subnet id. - `--amazonec2-security-group`: AWS VPC security group name. @@ -81,7 +71,7 @@ Environment variables and default values: | `--amazonec2-session-token` | `AWS_SESSION_TOKEN` | - | | `--amazonec2-ami` | `AWS_AMI` | `ami-5f709f34` | | `--amazonec2-region` | `AWS_DEFAULT_REGION` | `us-east-1` | -| **`--amazonec2-vpc-id`** | `AWS_VPC_ID` | - | +| `--amazonec2-vpc-id` | `AWS_VPC_ID` | - | | `--amazonec2-zone` | `AWS_ZONE` | `a` | | `--amazonec2-subnet-id` | `AWS_SUBNET_ID` | - | | `--amazonec2-security-group` | `AWS_SECURITY_GROUP` | `docker-machine` | @@ -109,3 +99,23 @@ Note that a security group will be created and associated to the host. This secu If you specify a security group yourself using the `--amazonec2-security-group` flag, the above ports will be checked and opened and the security group modified. If you want more ports to be opened, like application specific ports, use the aws console and modify the configuration manually. + + +## VPC ID +We determine your default vpc id at the start of a command. +In some cases, either because your account does not have a default vpc, or you don't want to use the default one, you can specify a vpc with the `--amazonec2-vpc-id` flag. + +To find the VPC ID: + +1. Login to the AWS console +2. Go to **Services -> VPC -> Your VPCs**. +3. Locate the VPC ID you want from the _VPC_ column. +4. Go to **Services -> VPC -> Subnets**. Examine the _Availability Zone_ column to verify that zone `a` exists and matches your VPC ID. + + For example, `us-east1-a` is in the `a` availability zone. If the `a` zone is not present, you can create a new subnet in that zone or specify a different zone when you create the machine. + +To create a machine with a non-default vpc-id: + + $ docker-machine create --driver amazonec2 --amazonec2-access-key AKI******* --amazonec2-secret-key 8T93C********* --amazonec2-vpc-id vpc-****** aws02 + +This example assumes the VPC ID was found in the `a` availability zone. Use the`--amazonec2-zone` flag to specify a zone other than the `a` zone. For example, `--amazonec2-zone c` signifies `us-east1-c`. From 6940dc33ca3636901e5c4ff22833ba637c182627 Mon Sep 17 00:00:00 2001 From: Jean-Laurent de Morlhon Date: Thu, 14 Jan 2016 11:34:00 +0100 Subject: [PATCH 4/5] If you got a const, use it. Signed-off-by: Jean-Laurent de Morlhon --- drivers/amazonec2/amazonec2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go index 33c0b0fffc..c86671c508 100644 --- a/drivers/amazonec2/amazonec2.go +++ b/drivers/amazonec2/amazonec2.go @@ -639,7 +639,7 @@ func (d *Driver) GetSSHHostname() (string, error) { func (d *Driver) GetSSHUsername() string { if d.SSHUser == "" { - d.SSHUser = "ubuntu" + d.SSHUser = defaultSSHUser } return d.SSHUser From c4e3480c74bb5110153a504bf51f8547b2aa8413 Mon Sep 17 00:00:00 2001 From: Jean-Laurent de Morlhon Date: Thu, 14 Jan 2016 09:44:14 +0100 Subject: [PATCH 5/5] Extract const for simple errors Signed-off-by: Jean-Laurent de Morlhon --- drivers/amazonec2/amazonec2.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go index c86671c508..66d3267242 100644 --- a/drivers/amazonec2/amazonec2.go +++ b/drivers/amazonec2/amazonec2.go @@ -47,8 +47,11 @@ const ( ) var ( - dockerPort = 2376 - swarmPort = 3376 + dockerPort = 2376 + swarmPort = 3376 + errorMissingAccessKeyOption = errors.New("amazonec2 driver requires the --amazonec2-access-key option") + errorMissingSecretKeyOption = errors.New("amazonec2 driver requires the --amazonec2-secret-key option") + errorNoVPCIdFound = errors.New("amazonec2 driver requires either the --amazonec2-subnet-id or --amazonec2-vpc-id option or an AWS Account with a default vpc-id") ) type Driver struct { @@ -282,22 +285,22 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { d.SetSwarmConfigFromFlags(flags) if d.AccessKey == "" { - return fmt.Errorf("amazonec2 driver requires the --amazonec2-access-key option") + return errorMissingAccessKeyOption } if d.SecretKey == "" { - return fmt.Errorf("amazonec2 driver requires the --amazonec2-secret-key option") + return errorMissingSecretKeyOption } if d.VpcId == "" { d.VpcId, err = d.getDefaultVPCId() if err != nil { - log.Errorf("couldn't determine your account Default VPC ID : %q", err) + log.Warnf("Couldn't determine your account Default VPC ID : %q", err) } } if d.SubnetId == "" && d.VpcId == "" { - return fmt.Errorf("amazonec2 driver requires either the --amazonec2-subnet-id or --amazonec2-vpc-id option or an AWS Account with a default vpc-id") + return errorNoVPCIdFound } if d.SubnetId != "" && d.VpcId != "" {