diff --git a/upup/pkg/fi/cloudup/awstasks/routetableassociation.go b/upup/pkg/fi/cloudup/awstasks/routetableassociation.go index fc7c1e1797..abbb2343f3 100644 --- a/upup/pkg/fi/cloudup/awstasks/routetableassociation.go +++ b/upup/pkg/fi/cloudup/awstasks/routetableassociation.go @@ -115,12 +115,14 @@ func findExistingRouteTableForSubnet(cloud awsup.AWSCloud, subnet *Subnet) (*ec2 return nil, fmt.Errorf("subnet ID not set") } + subnetID := fi.StringValue(subnet.ID) + request := &ec2.DescribeRouteTablesInput{ - Filters: []*ec2.Filter{awsup.NewEC2Filter("association.subnet-id", fi.StringValue(subnet.ID))}, + Filters: []*ec2.Filter{awsup.NewEC2Filter("association.subnet-id", subnetID)}, } response, err := cloud.EC2().DescribeRouteTables(request) if err != nil { - return nil, fmt.Errorf("error listing RouteTables: %v", err) + return nil, fmt.Errorf("error listing RouteTables for subnet %q: %v", subnetID, err) } if response == nil || len(response.RouteTables) == 0 { return nil, nil diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 553e0801f1..42d6a3a825 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -34,8 +34,17 @@ import ( "time" ) -const MaxDescribeTagsAttempts = 60 -const MaxCreateTagsAttempts = 60 +const DescribeTagsMaxAttempts = 120 +const DescribeTagsRetryInterval = 2 * time.Second +const DescribeTagsLogInterval = 10 // this is in "retry intervals" + +const CreateTagsMaxAttempts = 120 +const CreateTagsRetryInterval = 2 * time.Second +const CreateTagsLogInterval = 10 // this is in "retry intervals" + +const DeleteTagsMaxAttempts = 120 +const DeleteTagsRetryInterval = 2 * time.Second +const DeleteTagsLogInterval = 10 // this is in "retry intervals" const TagClusterName = "KubernetesCluster" @@ -196,12 +205,16 @@ func (c *awsCloudImplementation) GetTags(resourceId string) (map[string]string, response, err := c.EC2().DescribeTags(request) if err != nil { if isTagsEventualConsistencyError(err) { - if attempt > MaxDescribeTagsAttempts { + if attempt > DescribeTagsMaxAttempts { return nil, fmt.Errorf("Got retryable error while getting tags on %q, but retried too many times without success: %v", resourceId, err) } + if (attempt % DescribeTagsLogInterval) == 0 { + glog.Infof("waiting for eventual consistency while describing tags on %q", resourceId) + } + glog.V(2).Infof("will retry after encountering error getting tags on %q: %v", resourceId, err) - time.Sleep(2 * time.Second) + time.Sleep(DescribeTagsRetryInterval) continue } @@ -243,12 +256,16 @@ func (c *awsCloudImplementation) CreateTags(resourceId string, tags map[string]s _, err := c.EC2().CreateTags(request) if err != nil { if isTagsEventualConsistencyError(err) { - if attempt > MaxCreateTagsAttempts { + if attempt > CreateTagsMaxAttempts { return fmt.Errorf("Got retryable error while creating tags on %q, but retried too many times without success: %v", resourceId, err) } + if (attempt % CreateTagsLogInterval) == 0 { + glog.Infof("waiting for eventual consistency while creating tags on %q", resourceId) + } + glog.V(2).Infof("will retry after encountering error creating tags on %q: %v", resourceId, err) - time.Sleep(2 * time.Second) + time.Sleep(CreateTagsRetryInterval) continue } @@ -282,12 +299,16 @@ func (c *awsCloudImplementation) DeleteTags(resourceId string, tags map[string]s _, err := c.EC2().DeleteTags(request) if err != nil { if isTagsEventualConsistencyError(err) { - if attempt > MaxCreateTagsAttempts { + if attempt > DeleteTagsMaxAttempts { return fmt.Errorf("Got retryable error while deleting tags on %q, but retried too many times without success: %v", resourceId, err) } + if (attempt % DeleteTagsLogInterval) == 0 { + glog.Infof("waiting for eventual consistency while deleting tags on %q", resourceId) + } + glog.V(2).Infof("will retry after encountering error deleting tags on %q: %v", resourceId, err) - time.Sleep(2 * time.Second) + time.Sleep(DeleteTagsRetryInterval) continue }