Merge pull request #3021 from dhiltgen/harden_aws_mark2

Add retry support to AWS driver

docs/drivers/aws.md

@@ -71,6 +71,8 @@ You can use environment variables:
 -   `--amazonec2-monitoring`: Enable CloudWatch Monitoring.
 -   `--amazonec2-use-ebs-optimized-instance`: Create an EBS Optimized Instance, instance type must support it.
 -   `--amazonec2-ssh-keypath`: Path to Private Key file to use for instance. Matching public key with .pub extension should exist
+-   `--amazonec2-retries`:  Set retry count for recoverable failures (use -1 to disable)
+
 
 Environment variables and default values:
 
@@ -99,6 +101,7 @@ Environment variables and default values:
 | `--amazonec2-monitoring`                 | -                       | `false`          |
 | `--amazonec2-use-ebs-optimized-instance` | -                       | `false`          |
 | `--amazonec2-ssh-keypath`                | `AWS_SSH_KEYPATH`       | -                |
+| `--amazonec2-retries`                    | -                       | `5`              |
 
 ## Default AMIs
 

drivers/amazonec2/amazonec2.go

@@ -87,6 +87,7 @@ type Driver struct {
 	UseEbsOptimizedInstance bool
 	Monitoring              bool
 	SSHPrivateKeyPath       string
+	RetryCount              int
 }
 
 type clientFactory interface {
@@ -213,6 +214,11 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag {
 			Usage:  "SSH Key for Instance",
 			EnvVar: "AWS_SSH_KEYPATH",
 		},
+		mcnflag.IntFlag{
+			Name:  "amazonec2-retries",
+			Usage: "Set retry count for recoverable failures (use -1 to disable)",
+			Value: 5,
+		},
 	}
 }
 
@@ -247,6 +253,7 @@ func (d *Driver) buildClient() Ec2Client {
 	config = config.WithCredentials(d.awsCredentials.NewStaticCredentials(d.AccessKey, d.SecretKey, d.SessionToken))
 	config = config.WithLogger(alogger)
 	config = config.WithLogLevel(aws.LogDebugWithHTTPBody)
+	config = config.WithMaxRetries(d.RetryCount)
 	return ec2.New(session.New(config))
 }
 
@@ -291,6 +298,7 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
 	d.UseEbsOptimizedInstance = flags.Bool("amazonec2-use-ebs-optimized-instance")
 	d.SSHPrivateKeyPath = flags.String("amazonec2-ssh-keypath")
 	d.SetSwarmConfigFromFlags(flags)
+	d.RetryCount = flags.Int("amazonec2-retries")
 
 	if d.AccessKey == "" && d.SecretKey == "" {
 		credentials, err := d.awsCredentials.NewSharedCredentials("", "").Get()