From ebf05facc29e241cc6bb63236e6e7715be03dc8e Mon Sep 17 00:00:00 2001 From: msidwell Date: Thu, 17 Dec 2020 19:29:28 +0000 Subject: [PATCH] add gp3 volume default params add io2 case and correct IOPS minimum value check add gp3 case add io2 and gp3 parameter ratio validation logic add volumeThroughput parameter for disks that support it add volumeThroughput components throughout ebs structs add volumeThroughput to versioned api updated api machinery and crds apimachinery update --- k8s/crds/kops.k8s.io_clusters.yaml | 5 ++ pkg/apis/kops/cluster.go | 2 + pkg/apis/kops/v1alpha2/cluster.go | 2 + .../kops/v1alpha2/zz_generated.conversion.go | 2 + .../kops/v1alpha2/zz_generated.deepcopy.go | 5 ++ pkg/apis/kops/zz_generated.deepcopy.go | 5 ++ pkg/model/master_volumes.go | 47 ++++++++++++++----- upup/pkg/fi/cloudup/awstasks/ebsvolume.go | 7 +++ 8 files changed, 64 insertions(+), 11 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 166fcc863c..43abbeed0e 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -740,6 +740,11 @@ spec: size format: int32 type: integer + volumeThroughput: + description: Parameter for disks that support provisioned + throughput + format: int32 + type: integer volumeType: description: VolumeType is the underlying cloud storage class diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index ad2eabfdca..3354cfb1b3 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -531,6 +531,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index dd37b88157..74f7d0af86 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -528,6 +528,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 7f0b6b6e97..f4b0e2992e 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3059,6 +3059,7 @@ func autoConvert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec(in *EtcdMemberSp out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume @@ -3075,6 +3076,7 @@ func autoConvert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec(in *kops.EtcdMem out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index be142106b7..becc365089 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1395,6 +1395,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d7e6b0fdcc..f58f19119a 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1545,6 +1545,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go index e2f14afae8..5c28527d1b 100644 --- a/pkg/model/master_volumes.go +++ b/pkg/model/master_volumes.go @@ -37,11 +37,13 @@ import ( ) const ( - DefaultEtcdVolumeSize = 20 - DefaultAWSEtcdVolumeType = "gp2" - DefaultAWSEtcdVolumeIops = 100 - DefaultGCEEtcdVolumeType = "pd-ssd" - DefaultALIEtcdVolumeType = "cloud_ssd" + DefaultEtcdVolumeSize = 20 + DefaultAWSEtcdVolumeType = "gp2" + DefaultAWSEtcdVolumeIops = 100 + DefaultAWSEtcdVolumeGp3Iops = 3000 + DefaultAWSEtcdVolumeGp3Throughput = 125 + DefaultGCEEtcdVolumeType = "pd-ssd" + DefaultALIEtcdVolumeType = "cloud_ssd" ) // MasterVolumeBuilder builds master EBS volumes @@ -118,11 +120,23 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) volumeIops := fi.Int32Value(m.VolumeIops) + volumeThroughput := fi.Int32Value(m.VolumeThroughput) switch volumeType { case "io1": - if volumeIops <= 0 { + if volumeIops <= 100 { volumeIops = DefaultAWSEtcdVolumeIops } + case "io2": + if volumeIops < 100 { + volumeIops = DefaultAWSEtcdVolumeIops + } + case "gp3": + if volumeIops < 3000 { + volumeIops = DefaultAWSEtcdVolumeGp3Iops + } + if volumeThroughput < 125 { + volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput + } default: volumeType = DefaultAWSEtcdVolumeType } @@ -157,12 +171,23 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin Encrypted: fi.Bool(encrypted), Tags: tags, } - if volumeType == "io1" { - t.VolumeIops = i64(int64(volumeIops)) - + if strings.Contains(volumeType, "io") || volumeType == "gp3" { // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html - if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { - return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + t.VolumeIops = i64(int64(volumeIops)) + if volumeType == "io1" { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } else { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 500.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 500. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } + if volumeType == "gp3" { + t.VolumeThroughput = i64(int64(volumeThroughput)) + if float64(*t.VolumeThroughput)/float64(*t.VolumeIops) > 0.25 { + return fmt.Errorf("volumeThroughput to volumeIops ratio must be lower than 0.25. For %s ratio is %f", *t.Name, float64(*t.VolumeThroughput)/float64(*t.VolumeIops)) + } } } diff --git a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go index bddcd6bc53..a45acef7da 100644 --- a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go +++ b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go @@ -40,6 +40,7 @@ type EBSVolume struct { SizeGB *int64 Tags map[string]string VolumeIops *int64 + VolumeThroughput *int64 VolumeType *string } @@ -105,6 +106,7 @@ func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) { Encrypted: v.Encrypted, Name: e.Name, VolumeIops: v.Iops, + VolumeThroughput: v.Throughput, } actual.Tags = mapEC2TagsToMap(v.Tags) @@ -145,6 +147,7 @@ func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) e KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeVolume, e.Tags), } @@ -188,6 +191,7 @@ type terraformVolume struct { Size *int64 `json:"size,omitempty" cty:"size"` Type *string `json:"type,omitempty" cty:"type"` Iops *int64 `json:"iops,omitempty" cty:"iops"` + Throughput *int64 `json:"throughput,omitempty" cty:"throughput"` KmsKeyId *string `json:"kms_key_id,omitempty" cty:"kms_key_id"` Encrypted *bool `json:"encrypted,omitempty" cty:"encrypted"` Tags map[string]string `json:"tags,omitempty" cty:"tags"` @@ -199,6 +203,7 @@ func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: e.Tags, @@ -216,6 +221,7 @@ type cloudformationVolume struct { Size *int64 `json:"Size,omitempty"` Type *string `json:"VolumeType,omitempty"` Iops *int64 `json:"Iops,omitempty"` + Throughput *int64 `json:"Throughput,omitempty"` KmsKeyId *string `json:"KmsKeyId,omitempty"` Encrypted *bool `json:"Encrypted,omitempty"` Tags []cloudformationTag `json:"Tags,omitempty"` @@ -227,6 +233,7 @@ func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget, Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: buildCloudformationTags(e.Tags),