diff --git a/pkg/model/alimodel/scalinggroup.go b/pkg/model/alimodel/scalinggroup.go index 7f6a363428..f1716b9458 100644 --- a/pkg/model/alimodel/scalinggroup.go +++ b/pkg/model/alimodel/scalinggroup.go @@ -118,7 +118,7 @@ func (b *ScalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { SecurityGroup: b.LinkToSecurityGroup(ig.Spec.Role), RAMRole: b.LinkToRAMRole(ig.Spec.Role), - ImageId: s(ig.Spec.Image), + ImageID: s(ig.Spec.Image), InstanceType: s(instanceType), SystemDiskSize: i(int(volumeSize)), SystemDiskCategory: s(volumeType), diff --git a/upup/pkg/fi/cloudup/alitasks/BUILD.bazel b/upup/pkg/fi/cloudup/alitasks/BUILD.bazel index 1172e8abe9..149bc608e2 100644 --- a/upup/pkg/fi/cloudup/alitasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/alitasks/BUILD.bazel @@ -39,6 +39,7 @@ go_library( importpath = "k8s.io/kops/upup/pkg/fi/cloudup/alitasks", visibility = ["//visibility:public"], deps = [ + "//pkg/featureflag:go_default_library", "//pkg/pki:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/aliup:go_default_library", diff --git a/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go b/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go index 7d7185528d..35fd8107a8 100644 --- a/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go +++ b/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go @@ -20,12 +20,17 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math" + "sort" "strconv" + "strings" + "time" "github.com/denverdino/aliyungo/common" "github.com/denverdino/aliyungo/ess" "k8s.io/klog" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" "k8s.io/kops/upup/pkg/fi/cloudup/terraform" @@ -33,12 +38,27 @@ import ( //go:generate fitask -type=LaunchConfiguration -type LaunchConfiguration struct { - Lifecycle *fi.Lifecycle - Name *string - ConfigurationId *string +const dateFormat = "2006-01-02T15:04Z" - ImageId *string +// defaultRetainLaunchConfigurationCount is the number of launch configurations (matching the name prefix) that we should +// keep, we delete older ones +var defaultRetainLaunchConfigurationCount = 3 + +// RetainLaunchConfigurationCount returns the number of launch configurations to keep +func RetainLaunchConfigurationCount() int { + if featureflag.KeepLaunchConfigurations.Enabled() { + return math.MaxInt32 + } + return defaultRetainLaunchConfigurationCount +} + +// LaunchConfiguration is the specification for a launch configuration +type LaunchConfiguration struct { + Lifecycle *fi.Lifecycle + ID *string + Name *string + + ImageID *string InstanceType *string SystemDiskSize *int SystemDiskCategory *string @@ -55,7 +75,7 @@ type LaunchConfiguration struct { var _ fi.CompareWithID = &LaunchConfiguration{} func (l *LaunchConfiguration) CompareWithID() *string { - return l.ConfigurationId + return l.ID } func (l *LaunchConfiguration) Find(c *fi.Context) (*LaunchConfiguration, error) { @@ -64,40 +84,28 @@ func (l *LaunchConfiguration) Find(c *fi.Context) (*LaunchConfiguration, error) return nil, nil } - cloud := c.Cloud.(aliup.ALICloud) - - describeScalingConfigurationsArgs := &ess.DescribeScalingConfigurationsArgs{ - RegionId: common.Region(cloud.Region()), - } - - if l.ScalingGroup != nil && l.ScalingGroup.ScalingGroupId != nil { - describeScalingConfigurationsArgs.ScalingGroupId = fi.StringValue(l.ScalingGroup.ScalingGroupId) - } - - configList, _, err := cloud.EssClient().DescribeScalingConfigurations(describeScalingConfigurationsArgs) + configurations, err := l.findLaunchConfigurations(c) if err != nil { return nil, fmt.Errorf("error finding ScalingConfigurations: %v", err) } // No ScalingConfigurations with specified Name. - if len(configList) == 0 { + if len(configurations) == 0 { klog.V(2).Infof("can't found matching LaunchConfiguration: %q", fi.StringValue(l.Name)) return nil, nil } - if len(configList) > 1 { - return nil, fmt.Errorf("found multiple LaunchConfiguration with name: %q", fi.StringValue(l.Name)) - } - klog.V(2).Infof("found matching LaunchConfiguration: %q", fi.StringValue(l.Name)) - lc := configList[0] + lc := configurations[len(configurations)-1] + + klog.V(2).Infof("found matching LaunchConfiguration: %q", lc.ScalingConfigurationName) actual := &LaunchConfiguration{ - ImageId: fi.String(lc.ImageId), + Name: l.Name, + ID: fi.String(lc.ScalingConfigurationId), + ImageID: fi.String(lc.ImageId), InstanceType: fi.String(lc.InstanceType), SystemDiskSize: fi.Int(lc.SystemDiskSize), SystemDiskCategory: fi.String(string(lc.SystemDiskCategory)), - ConfigurationId: fi.String(lc.ScalingConfigurationId), - Name: fi.String(lc.ScalingConfigurationName), } if lc.KeyPairName != "" { @@ -139,6 +147,64 @@ func (l *LaunchConfiguration) Find(c *fi.Context) (*LaunchConfiguration, error) return actual, nil } +func (l *LaunchConfiguration) findLaunchConfigurations(c *fi.Context) ([]*ess.ScalingConfigurationItemType, error) { + cloud := c.Cloud.(aliup.ALICloud) + prefix := *l.Name + "-" + + var configurations []*ess.ScalingConfigurationItemType + + pageNumber := 1 + pageSize := 50 + for { + describeSCArgs := &ess.DescribeScalingConfigurationsArgs{ + RegionId: common.Region(cloud.Region()), + Pagination: common.Pagination{ + PageNumber: pageNumber, + PageSize: pageSize, + }, + } + + if l.ScalingGroup != nil && l.ScalingGroup.ScalingGroupId != nil { + describeSCArgs.ScalingGroupId = fi.StringValue(l.ScalingGroup.ScalingGroupId) + } + + configs, _, err := cloud.EssClient().DescribeScalingConfigurations(describeSCArgs) + if err != nil { + return nil, fmt.Errorf("error finding ScalingConfigurations: %v", err) + } + + for _, c := range configs { + if strings.HasPrefix(c.ScalingConfigurationName, prefix) { + + // Verify the CreationTime is parseble here, so we can ignore errors when sorting + _, err := time.Parse(dateFormat, c.CreationTime) + if err != nil { + return nil, fmt.Errorf("error parse CreationTime %s: %v", c.CreationTime, err) + } + + cc := c // Copy the pointer during iteration + configurations = append(configurations, &cc) + } + } + + if len(configs) < pageSize { + break + } else { + pageNumber++ + } + + klog.V(4).Infof("Describing ScalingConfigurations page %v...", pageNumber) + } + + sort.Slice(configurations, func(i, j int) bool { + ti, _ := time.Parse(dateFormat, configurations[i].CreationTime) + tj, _ := time.Parse(dateFormat, configurations[j].CreationTime) + return ti.UnixNano() < tj.UnixNano() + }) + + return configurations, nil +} + func (l *LaunchConfiguration) Run(c *fi.Context) error { c.Cloud.(aliup.ALICloud).AddClusterTags(l.Tags) return fi.DefaultDeltaRunMethod(l, c) @@ -146,14 +212,14 @@ func (l *LaunchConfiguration) Run(c *fi.Context) error { func (_ *LaunchConfiguration) CheckChanges(a, e, changes *LaunchConfiguration) error { //Configuration can not be modified, we need to create a new one - if e.Name == nil { return fi.RequiredField("Name") } - if e.ImageId == nil { + if e.ImageID == nil { return fi.RequiredField("ImageId") } + if e.InstanceType == nil { return fi.RequiredField("InstanceType") } @@ -162,13 +228,13 @@ func (_ *LaunchConfiguration) CheckChanges(a, e, changes *LaunchConfiguration) e } func (_ *LaunchConfiguration) RenderALI(t *aliup.ALIAPITarget, a, e, changes *LaunchConfiguration) error { - - klog.V(2).Infof("Creating LaunchConfiguration for ScalingGroup:%q", fi.StringValue(e.ScalingGroup.ScalingGroupId)) + launchConfigurationName := *e.Name + "-" + fi.BuildTimestampString() + klog.V(2).Infof("Creating LaunchConfiguration with name:%q", launchConfigurationName) createScalingConfiguration := &ess.CreateScalingConfigurationArgs{ + ScalingConfigurationName: launchConfigurationName, ScalingGroupId: fi.StringValue(e.ScalingGroup.ScalingGroupId), - ScalingConfigurationName: fi.StringValue(e.Name), - ImageId: fi.StringValue(e.ImageId), + ImageId: fi.StringValue(e.ImageID), InstanceType: fi.StringValue(e.InstanceType), SecurityGroupId: fi.StringValue(e.SecurityGroup.SecurityGroupId), SystemDisk_Size: common.UnderlineString(strconv.Itoa(fi.IntValue(e.SystemDiskSize))), @@ -194,7 +260,7 @@ func (_ *LaunchConfiguration) RenderALI(t *aliup.ALIAPITarget, a, e, changes *La if e.Tags != nil { tagItem, err := json.Marshal(e.Tags) if err != nil { - return fmt.Errorf("error rendering ScalingLaunchConfiguration Tags: %v", err) + return fmt.Errorf("error rendering LaunchConfiguration Tags: %v", err) } createScalingConfiguration.Tags = string(tagItem) } @@ -203,12 +269,11 @@ func (_ *LaunchConfiguration) RenderALI(t *aliup.ALIAPITarget, a, e, changes *La if err != nil { return fmt.Errorf("error creating scalingConfiguration: %v", err) } - e.ConfigurationId = fi.String(createScalingConfigurationResponse.ScalingConfigurationId) + e.ID = fi.String(createScalingConfigurationResponse.ScalingConfigurationId) // Disable ScalingGroup, used to bind scalingConfig, we should execute EnableScalingGroup in the task LaunchConfiguration // If the ScalingGroup is active, we can not execute EnableScalingGroup. if e.ScalingGroup.Active != nil && fi.BoolValue(e.ScalingGroup.Active) { - klog.V(2).Infof("Disabling LoadBalancer with id:%q", fi.StringValue(e.ScalingGroup.ScalingGroupId)) disableScalingGroupArgs := &ess.DisableScalingGroupArgs{ @@ -223,7 +288,7 @@ func (_ *LaunchConfiguration) RenderALI(t *aliup.ALIAPITarget, a, e, changes *La //Enable this configuration enableScalingGroupArgs := &ess.EnableScalingGroupArgs{ ScalingGroupId: fi.StringValue(e.ScalingGroup.ScalingGroupId), - ActiveScalingConfigurationId: fi.StringValue(e.ConfigurationId), + ActiveScalingConfigurationId: fi.StringValue(e.ID), } klog.V(2).Infof("Enabling new LaunchConfiguration of LoadBalancer with id:%q", fi.StringValue(e.ScalingGroup.ScalingGroupId)) @@ -257,7 +322,7 @@ func (_ *LaunchConfiguration) RenderTerraform(t *terraform.TerraformTarget, a, e userData := base64.StdEncoding.EncodeToString(data) tf := &terraformLaunchConfiguration{ - ImageID: e.ImageId, + ImageID: e.ImageID, InstanceType: e.InstanceType, SystemDiskCategory: e.SystemDiskCategory, UserData: &userData, @@ -274,3 +339,68 @@ func (_ *LaunchConfiguration) RenderTerraform(t *terraform.TerraformTarget, a, e func (l *LaunchConfiguration) TerraformLink() *terraform.Literal { return terraform.LiteralProperty("alicloud_ess_scaling_configuration", fi.StringValue(l.Name), "id") } + +// deleteLaunchConfiguration tracks a LaunchConfiguration that we're going to delete +// It implements fi.Deletion +type deleteLaunchConfiguration struct { + lc *ess.ScalingConfigurationItemType +} + +var _ fi.Deletion = &deleteLaunchConfiguration{} + +func (d *deleteLaunchConfiguration) TaskName() string { + return "LaunchConfiguration" +} + +func (d *deleteLaunchConfiguration) Item() string { + return d.lc.ScalingConfigurationName +} + +func (d *deleteLaunchConfiguration) Delete(t fi.Target) error { + klog.V(2).Infof("deleting launch configuration %v", d) + + aliTarget, ok := t.(*aliup.ALIAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + request := &ess.DeleteScalingConfigurationArgs{ + ScalingConfigurationId: d.lc.ScalingConfigurationId, + } + + id := request.ScalingConfigurationId + klog.V(2).Infof("Calling ESS DeleteScalingConfiguration for %s", id) + _, err := aliTarget.Cloud.EssClient().DeleteScalingConfiguration(request) + if err != nil { + return fmt.Errorf("error deleting ESS LaunchConfiguration %s: %v", id, err) + } + + return nil +} + +func (d *deleteLaunchConfiguration) String() string { + return d.TaskName() + "-" + d.Item() +} + +func (e *LaunchConfiguration) FindDeletions(c *fi.Context) ([]fi.Deletion, error) { + var removals []fi.Deletion + + configurations, err := e.findLaunchConfigurations(c) + if err != nil { + return nil, err + } + + if len(configurations) <= RetainLaunchConfigurationCount() { + return nil, nil + } + + configurations = configurations[:len(configurations)-RetainLaunchConfigurationCount()] + + for _, configuration := range configurations { + removals = append(removals, &deleteLaunchConfiguration{lc: configuration}) + } + + klog.V(2).Infof("will delete launch configurations: %v", removals) + + return removals, nil +} diff --git a/upup/pkg/fi/cloudup/aliup/ali_cloud.go b/upup/pkg/fi/cloudup/aliup/ali_cloud.go index 5706038b43..f058e01679 100644 --- a/upup/pkg/fi/cloudup/aliup/ali_cloud.go +++ b/upup/pkg/fi/cloudup/aliup/ali_cloud.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "os" + "time" "k8s.io/klog" @@ -142,7 +143,41 @@ func (c *aliCloudImplementation) DeleteGroup(g *cloudinstances.CloudInstanceGrou } func (c *aliCloudImplementation) DeleteInstance(i *cloudinstances.CloudInstanceGroupMember) error { - return errors.New("DeleteInstance not implemented on aliCloud") + id := i.ID + if id == "" { + return fmt.Errorf("id was not set on CloudInstanceGroupMember: %v", i) + } + + if err := c.EcsClient().StopInstance(id, false); err != nil { + return fmt.Errorf("error stopping instance %q: %v", id, err) + } + + // Wait for 3 min to stop the instance + for i := 0; i < 36; i++ { + ins, err := c.EcsClient().DescribeInstanceAttribute(id) + if err != nil { + return fmt.Errorf("error describing instance %q: %v", id, err) + } + + klog.V(8).Infof("stopping Alicloud ecs instance %q, current Status: %q", id, ins.Status) + time.Sleep(time.Second * 5) + + if ins.Status == ecs.Stopped { + break + } + + if i == 35 { + return fmt.Errorf("fail to stop ecs instance %s in 3 mins", id) + } + } + + if err := c.EcsClient().DeleteInstance(id); err != nil { + return fmt.Errorf("error deleting instance %q: %v", id, err) + } + + klog.V(8).Infof("deleted Alicloud ecs instance %q", id) + + return nil } func (c *aliCloudImplementation) FindVPCInfo(id string) (*fi.VPCInfo, error) {