mirror of https://github.com/kubernetes/kops.git
505 lines
16 KiB
Go
505 lines
16 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package awsmodel
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"k8s.io/klog/v2"
|
|
"k8s.io/kops/pkg/apis/kops"
|
|
"k8s.io/kops/pkg/featureflag"
|
|
"k8s.io/kops/pkg/model"
|
|
"k8s.io/kops/pkg/model/defaults"
|
|
"k8s.io/kops/upup/pkg/fi"
|
|
"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
|
|
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
|
|
)
|
|
|
|
const (
|
|
// DefaultVolumeType is the default volume type
|
|
DefaultVolumeType = ec2.VolumeTypeGp3
|
|
// DefaultLegacyVolumeType is the default volume type when using LaunchConfigurations
|
|
DefaultLegacyVolumeType = ec2.VolumeTypeGp2
|
|
// DefaultVolumeIonIops is the default volume IOPS when volume type is io1 or io2
|
|
DefaultVolumeIonIops = 100
|
|
// DefaultVolumeGp3Iops is the default volume IOPS when volume type is gp3
|
|
DefaultVolumeGp3Iops = 3000
|
|
// DefaultVolumeGp3Throughput is the default volume throughput when volume type is gp3
|
|
DefaultVolumeGp3Throughput = 125
|
|
// DefaultVolumeDeleteOnTermination is the default volume behavior after instance termination
|
|
DefaultVolumeDeleteOnTermination = true
|
|
// DefaultVolumeEncryption is the default volume encryption behavior
|
|
DefaultVolumeEncryption = true
|
|
)
|
|
|
|
// AutoscalingGroupModelBuilder configures AutoscalingGroup objects
|
|
type AutoscalingGroupModelBuilder struct {
|
|
*AWSModelContext
|
|
|
|
BootstrapScriptBuilder *model.BootstrapScriptBuilder
|
|
Lifecycle *fi.Lifecycle
|
|
SecurityLifecycle *fi.Lifecycle
|
|
Cluster *kops.Cluster
|
|
}
|
|
|
|
var _ fi.ModelBuilder = &AutoscalingGroupModelBuilder{}
|
|
|
|
// Build is responsible for constructing the aws autoscaling group from the kops spec
|
|
func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
|
|
for _, ig := range b.InstanceGroups {
|
|
name := b.AutoscalingGroupName(ig)
|
|
|
|
if featureflag.SpotinstHybrid.Enabled() {
|
|
if HybridInstanceGroup(ig) {
|
|
klog.V(2).Infof("Skipping instance group: %q", name)
|
|
continue
|
|
}
|
|
}
|
|
|
|
task, err := b.buildLaunchTemplateTask(c, name, ig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.AddTask(task)
|
|
|
|
// @step: now lets build the autoscaling group task
|
|
tsk, err := b.buildAutoScalingGroupTask(c, name, ig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tsk.LaunchTemplate = task
|
|
c.AddTask(tsk)
|
|
|
|
warmPool := b.Cluster.Spec.WarmPool.ResolveDefaults(ig)
|
|
{
|
|
enabled := fi.Bool(warmPool.IsEnabled())
|
|
warmPoolTask := &awstasks.WarmPool{
|
|
Name: &name,
|
|
Enabled: enabled,
|
|
}
|
|
if warmPool.IsEnabled() {
|
|
warmPoolTask.MinSize = warmPool.MinSize
|
|
warmPoolTask.MaxSize = warmPool.MaxSize
|
|
|
|
if warmPool.EnableLifecycleHook {
|
|
hookName := "kops-warmpool"
|
|
name := fmt.Sprintf("%s-%s", hookName, ig.GetName())
|
|
|
|
lifecyleTask := &awstasks.AutoscalingLifecycleHook{
|
|
ID: aws.String(name),
|
|
Name: aws.String(name),
|
|
HookName: aws.String(hookName),
|
|
Lifecycle: b.Lifecycle,
|
|
AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
|
|
DefaultResult: aws.String("ABANDON"),
|
|
// We let nodeup have 10 min to complete. Normally this should happen much faster,
|
|
// but CP nodes need 5 min or so to start on new clusters, and we need to wait for that.
|
|
HeartbeatTimeout: aws.Int64(600),
|
|
LifecycleTransition: aws.String("autoscaling:EC2_INSTANCE_LAUNCHING"),
|
|
}
|
|
|
|
c.AddTask(lifecyleTask)
|
|
|
|
}
|
|
|
|
}
|
|
c.AddTask(warmPoolTask)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// buildLaunchTemplateTask is responsible for creating the template task into the aws model
|
|
func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchTemplate, error) {
|
|
lc, err := b.buildLaunchTemplateHelper(c, name, ig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tags, err := b.CloudTagsForInstanceGroup(ig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error building cloud tags: %v", err)
|
|
}
|
|
|
|
lt := &awstasks.LaunchTemplate{
|
|
Name: fi.String(name),
|
|
Lifecycle: b.Lifecycle,
|
|
CPUCredits: fi.String(fi.StringValue(ig.Spec.CPUCredits)),
|
|
HTTPPutResponseHopLimit: fi.Int64(1),
|
|
HTTPTokens: fi.String(ec2.LaunchTemplateHttpTokensStateOptional),
|
|
IAMInstanceProfile: lc.IAMInstanceProfile,
|
|
ImageID: lc.ImageID,
|
|
InstanceMonitoring: lc.InstanceMonitoring,
|
|
InstanceType: lc.InstanceType,
|
|
RootVolumeOptimization: lc.RootVolumeOptimization,
|
|
RootVolumeSize: lc.RootVolumeSize,
|
|
RootVolumeIops: lc.RootVolumeIops,
|
|
RootVolumeType: lc.RootVolumeType,
|
|
RootVolumeEncryption: lc.RootVolumeEncryption,
|
|
SSHKey: lc.SSHKey,
|
|
SecurityGroups: lc.SecurityGroups,
|
|
Tags: tags,
|
|
Tenancy: lc.Tenancy,
|
|
UserData: lc.UserData,
|
|
}
|
|
|
|
{
|
|
// @step: check the subnets are ok and pull together an array for us
|
|
subnets, err := b.GatherSubnets(ig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// @step: check if we can add an public ip to this subnet
|
|
switch subnets[0].Type {
|
|
case kops.SubnetTypePublic, kops.SubnetTypeUtility:
|
|
lt.AssociatePublicIP = fi.Bool(true)
|
|
if ig.Spec.AssociatePublicIP != nil {
|
|
lt.AssociatePublicIP = ig.Spec.AssociatePublicIP
|
|
}
|
|
case kops.SubnetTypePrivate:
|
|
lt.AssociatePublicIP = fi.Bool(false)
|
|
}
|
|
}
|
|
|
|
// @step: add any additional block devices
|
|
for i := range ig.Spec.Volumes {
|
|
x := &ig.Spec.Volumes[i]
|
|
if x.Type == "" {
|
|
x.Type = DefaultVolumeType
|
|
}
|
|
if x.Type == ec2.VolumeTypeIo1 || x.Type == ec2.VolumeTypeIo2 {
|
|
if x.Iops == nil {
|
|
x.Iops = fi.Int64(DefaultVolumeIonIops)
|
|
}
|
|
} else if x.Type == ec2.VolumeTypeGp3 {
|
|
if x.Iops == nil {
|
|
x.Iops = fi.Int64(DefaultVolumeGp3Iops)
|
|
}
|
|
if x.Throughput == nil {
|
|
x.Throughput = fi.Int64(DefaultVolumeGp3Throughput)
|
|
}
|
|
} else {
|
|
x.Iops = nil
|
|
}
|
|
deleteOnTermination := DefaultVolumeDeleteOnTermination
|
|
if x.DeleteOnTermination != nil {
|
|
deleteOnTermination = fi.BoolValue(x.DeleteOnTermination)
|
|
}
|
|
encryption := DefaultVolumeEncryption
|
|
if x.Encrypted != nil {
|
|
encryption = fi.BoolValue(x.Encrypted)
|
|
}
|
|
lt.BlockDeviceMappings = append(lt.BlockDeviceMappings, &awstasks.BlockDeviceMapping{
|
|
DeviceName: fi.String(x.Device),
|
|
EbsDeleteOnTermination: fi.Bool(deleteOnTermination),
|
|
EbsEncrypted: fi.Bool(encryption),
|
|
EbsKmsKey: x.Key,
|
|
EbsVolumeIops: x.Iops,
|
|
EbsVolumeSize: fi.Int64(x.Size),
|
|
EbsVolumeThroughput: x.Throughput,
|
|
EbsVolumeType: fi.String(x.Type),
|
|
})
|
|
}
|
|
|
|
if ig.Spec.InstanceMetadata != nil && ig.Spec.InstanceMetadata.HTTPPutResponseHopLimit != nil {
|
|
lt.HTTPPutResponseHopLimit = ig.Spec.InstanceMetadata.HTTPPutResponseHopLimit
|
|
}
|
|
|
|
if ig.Spec.InstanceMetadata != nil && ig.Spec.InstanceMetadata.HTTPTokens != nil {
|
|
lt.HTTPTokens = ig.Spec.InstanceMetadata.HTTPTokens
|
|
}
|
|
|
|
// When using a MixedInstances ASG, AWS requires the SpotPrice be defined on the ASG
|
|
// rather than the LaunchTemplate or else it returns this error:
|
|
// You cannot use a launch template that is set to request Spot Instances (InstanceMarketOptions)
|
|
// when you configure an Auto Scaling group with a mixed instances policy.
|
|
if ig.Spec.MixedInstancesPolicy == nil && ig.Spec.MaxPrice != nil {
|
|
lt.SpotPrice = ig.Spec.MaxPrice
|
|
} else {
|
|
lt.SpotPrice = fi.String("")
|
|
}
|
|
if ig.Spec.SpotDurationInMinutes != nil {
|
|
lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes
|
|
}
|
|
if ig.Spec.InstanceInterruptionBehavior != nil {
|
|
lt.InstanceInterruptionBehavior = ig.Spec.InstanceInterruptionBehavior
|
|
}
|
|
if fi.BoolValue(ig.Spec.RootVolumeEncryption) && ig.Spec.RootVolumeEncryptionKey != nil {
|
|
lt.RootVolumeKmsKey = ig.Spec.RootVolumeEncryptionKey
|
|
} else {
|
|
lt.RootVolumeKmsKey = fi.String("")
|
|
}
|
|
if fi.StringValue(ig.Spec.RootVolumeType) == "" {
|
|
lt.RootVolumeType = fi.String(DefaultVolumeType)
|
|
}
|
|
if fi.StringValue(lt.RootVolumeType) == ec2.VolumeTypeGp3 {
|
|
if fi.Int32Value(ig.Spec.RootVolumeIops) < 3000 {
|
|
lt.RootVolumeIops = fi.Int64(int64(DefaultVolumeGp3Iops))
|
|
} else {
|
|
lt.RootVolumeIops = fi.Int64(int64(fi.Int32Value(ig.Spec.RootVolumeIops)))
|
|
}
|
|
if fi.Int32Value(ig.Spec.RootVolumeThroughput) < 125 {
|
|
lt.RootVolumeThroughput = fi.Int64(int64(DefaultVolumeGp3Throughput))
|
|
} else {
|
|
lt.RootVolumeThroughput = fi.Int64(int64(fi.Int32Value(ig.Spec.RootVolumeThroughput)))
|
|
}
|
|
}
|
|
|
|
return lt, nil
|
|
}
|
|
|
|
// buildLaunchTemplateHelper is responsible for building a launch configuration task into the model
|
|
func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateHelper(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchTemplate, error) {
|
|
// @step: lets add the root volume settings
|
|
volumeSize, err := defaults.DefaultInstanceGroupVolumeSize(ig.Spec.Role)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if fi.Int32Value(ig.Spec.RootVolumeSize) > 0 {
|
|
volumeSize = fi.Int32Value(ig.Spec.RootVolumeSize)
|
|
}
|
|
|
|
volumeType := fi.StringValue(ig.Spec.RootVolumeType)
|
|
if volumeType == "" {
|
|
volumeType = DefaultLegacyVolumeType
|
|
}
|
|
|
|
rootVolumeEncryption := DefaultVolumeEncryption
|
|
if ig.Spec.RootVolumeEncryption != nil {
|
|
rootVolumeEncryption = fi.BoolValue(ig.Spec.RootVolumeEncryption)
|
|
}
|
|
|
|
// @step: if required we add the override for the security group for this instancegroup
|
|
sgLink := b.LinkToSecurityGroup(ig.Spec.Role)
|
|
if ig.Spec.SecurityGroupOverride != nil {
|
|
sgName := fmt.Sprintf("%v-%v", fi.StringValue(ig.Spec.SecurityGroupOverride), ig.Spec.Role)
|
|
sgLink = &awstasks.SecurityGroup{
|
|
ID: ig.Spec.SecurityGroupOverride,
|
|
Name: &sgName,
|
|
Shared: fi.Bool(true),
|
|
}
|
|
}
|
|
|
|
// @step: add the iam instance profile
|
|
link, err := b.LinkToIAMInstanceProfile(ig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to find IAM profile link for instance group %q: %w", ig.ObjectMeta.Name, err)
|
|
}
|
|
|
|
t := &awstasks.LaunchTemplate{
|
|
Name: fi.String(name),
|
|
Lifecycle: b.Lifecycle,
|
|
IAMInstanceProfile: link,
|
|
ImageID: fi.String(ig.Spec.Image),
|
|
InstanceMonitoring: ig.Spec.DetailedInstanceMonitoring,
|
|
InstanceType: fi.String(strings.Split(ig.Spec.MachineType, ",")[0]),
|
|
RootVolumeOptimization: ig.Spec.RootVolumeOptimization,
|
|
RootVolumeSize: fi.Int64(int64(volumeSize)),
|
|
RootVolumeType: fi.String(volumeType),
|
|
RootVolumeEncryption: fi.Bool(rootVolumeEncryption),
|
|
SecurityGroups: []*awstasks.SecurityGroup{sgLink},
|
|
}
|
|
|
|
if ig.HasAPIServer() &&
|
|
b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
|
|
for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups {
|
|
sgTask := &awstasks.SecurityGroup{
|
|
ID: fi.String(id),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
Name: fi.String("nlb-" + id),
|
|
Shared: fi.Bool(true),
|
|
}
|
|
if err := c.EnsureTask(sgTask); err != nil {
|
|
return nil, err
|
|
}
|
|
t.SecurityGroups = append(t.SecurityGroups, sgTask)
|
|
}
|
|
}
|
|
|
|
if volumeType == ec2.VolumeTypeIo1 || volumeType == ec2.VolumeTypeIo2 {
|
|
if fi.Int32Value(ig.Spec.RootVolumeIops) < 100 {
|
|
t.RootVolumeIops = fi.Int64(int64(DefaultVolumeIonIops))
|
|
} else {
|
|
t.RootVolumeIops = fi.Int64(int64(fi.Int32Value(ig.Spec.RootVolumeIops)))
|
|
}
|
|
}
|
|
|
|
if ig.Spec.Tenancy != "" {
|
|
t.Tenancy = fi.String(ig.Spec.Tenancy)
|
|
}
|
|
|
|
// @step: add any additional security groups to the instancegroup
|
|
for _, id := range ig.Spec.AdditionalSecurityGroups {
|
|
sgTask := &awstasks.SecurityGroup{
|
|
ID: fi.String(id),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
Name: fi.String(id),
|
|
Shared: fi.Bool(true),
|
|
}
|
|
if err := c.EnsureTask(sgTask); err != nil {
|
|
return nil, err
|
|
}
|
|
t.SecurityGroups = append(t.SecurityGroups, sgTask)
|
|
}
|
|
|
|
if b.AWSModelContext.UseSSHKey() {
|
|
if t.SSHKey, err = b.LinkToSSHKey(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// @step: add the instancegroup userdata
|
|
if t.UserData, err = b.BootstrapScriptBuilder.ResourceNodeUp(c, ig); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return t, nil
|
|
}
|
|
|
|
// buildAutoscalingGroupTask is responsible for building the autoscaling task into the model
|
|
func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.AutoscalingGroup, error) {
|
|
|
|
t := &awstasks.AutoscalingGroup{
|
|
Name: fi.String(name),
|
|
Lifecycle: b.Lifecycle,
|
|
|
|
Granularity: fi.String("1Minute"),
|
|
Metrics: []string{
|
|
"GroupDesiredCapacity",
|
|
"GroupInServiceInstances",
|
|
"GroupMaxSize",
|
|
"GroupMinSize",
|
|
"GroupPendingInstances",
|
|
"GroupStandbyInstances",
|
|
"GroupTerminatingInstances",
|
|
"GroupTotalInstances",
|
|
},
|
|
}
|
|
|
|
minSize := fi.Int64(1)
|
|
maxSize := fi.Int64(1)
|
|
if ig.Spec.MinSize != nil {
|
|
minSize = fi.Int64(int64(*ig.Spec.MinSize))
|
|
} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
|
|
minSize = fi.Int64(2)
|
|
}
|
|
if ig.Spec.MaxSize != nil {
|
|
maxSize = fi.Int64(int64(*ig.Spec.MaxSize))
|
|
} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
|
|
maxSize = fi.Int64(2)
|
|
}
|
|
|
|
t.MinSize = minSize
|
|
t.MaxSize = maxSize
|
|
|
|
subnets, err := b.GatherSubnets(ig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(subnets) == 0 {
|
|
return nil, fmt.Errorf("could not determine any subnets for InstanceGroup %q; subnets was %s", ig.ObjectMeta.Name, ig.Spec.Subnets)
|
|
}
|
|
for _, subnet := range subnets {
|
|
t.Subnets = append(t.Subnets, b.LinkToSubnet(subnet))
|
|
}
|
|
|
|
tags, err := b.CloudTagsForInstanceGroup(ig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error building cloud tags: %v", err)
|
|
}
|
|
t.Tags = tags
|
|
|
|
processes := []string{}
|
|
processes = append(processes, ig.Spec.SuspendProcesses...)
|
|
t.SuspendProcesses = &processes
|
|
|
|
t.InstanceProtection = ig.Spec.InstanceProtection
|
|
|
|
t.LoadBalancers = []*awstasks.ClassicLoadBalancer{}
|
|
t.TargetGroups = []*awstasks.TargetGroup{}
|
|
|
|
// Spotinst handles load balancer attachments internally, so there's no
|
|
// need to create separate attachments for both managed (+Spotinst) and
|
|
// hybrid (+SpotinstHybrid) instance groups.
|
|
if !featureflag.Spotinst.Enabled() ||
|
|
(featureflag.SpotinstHybrid.Enabled() && !HybridInstanceGroup(ig)) {
|
|
if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
|
|
if b.UseNetworkLoadBalancer() {
|
|
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
|
|
if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
|
|
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tls"))
|
|
}
|
|
} else {
|
|
t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("api"))
|
|
}
|
|
}
|
|
|
|
if ig.Spec.Role == kops.InstanceGroupRoleBastion {
|
|
t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("bastion"))
|
|
}
|
|
}
|
|
|
|
for _, extLB := range ig.Spec.ExternalLoadBalancers {
|
|
if extLB.LoadBalancerName != nil {
|
|
lb := &awstasks.ClassicLoadBalancer{
|
|
Name: extLB.LoadBalancerName,
|
|
LoadBalancerName: extLB.LoadBalancerName,
|
|
Shared: fi.Bool(true),
|
|
}
|
|
t.LoadBalancers = append(t.LoadBalancers, lb)
|
|
c.EnsureTask(lb)
|
|
}
|
|
|
|
if extLB.TargetGroupARN != nil {
|
|
targetGroupName, err := awsup.GetTargetGroupNameFromARN(fi.StringValue(extLB.TargetGroupARN))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tg := &awstasks.TargetGroup{
|
|
Name: fi.String(name + "-" + targetGroupName),
|
|
ARN: extLB.TargetGroupARN,
|
|
Shared: fi.Bool(true),
|
|
}
|
|
t.TargetGroups = append(t.TargetGroups, tg)
|
|
c.AddTask(tg)
|
|
}
|
|
}
|
|
sort.Stable(awstasks.OrderLoadBalancersByName(t.LoadBalancers))
|
|
sort.Stable(awstasks.OrderTargetGroupsByName(t.TargetGroups))
|
|
|
|
// @step: are we using a mixed instance policy
|
|
if ig.Spec.MixedInstancesPolicy != nil {
|
|
spec := ig.Spec.MixedInstancesPolicy
|
|
|
|
t.MixedInstanceOverrides = spec.Instances
|
|
t.MixedOnDemandAboveBase = spec.OnDemandAboveBase
|
|
t.MixedOnDemandAllocationStrategy = spec.OnDemandAllocationStrategy
|
|
t.MixedOnDemandBase = spec.OnDemandBase
|
|
t.MixedSpotAllocationStrategy = spec.SpotAllocationStrategy
|
|
t.MixedSpotInstancePools = spec.SpotInstancePools
|
|
t.MixedSpotMaxPrice = ig.Spec.MaxPrice
|
|
}
|
|
|
|
return t, nil
|
|
}
|