From 3b51ce413b6a2a74f6a66ca944165f021a21cbbd Mon Sep 17 00:00:00 2001 From: msidwell Date: Thu, 17 Dec 2020 19:29:28 +0000 Subject: [PATCH] add gp3 volume default params add io2 case and correct IOPS minimum value check add gp3 case add io2 and gp3 parameter ratio validation logic add volumeThroughput parameter for disks that support it add volumeThroughput components throughout ebs structs add volumeThroughput to versioned api updated api machinery and crds apimachinery update --- k8s/crds/kops.k8s.io_clusters.yaml | 4 ++ pkg/apis/kops/cluster.go | 2 + pkg/apis/kops/v1alpha2/cluster.go | 2 + .../kops/v1alpha2/zz_generated.conversion.go | 2 + .../kops/v1alpha2/zz_generated.deepcopy.go | 5 ++ pkg/apis/kops/zz_generated.deepcopy.go | 5 ++ pkg/model/master_volumes.go | 47 ++++++++++++++----- upup/pkg/fi/cloudup/awstasks/ebsvolume.go | 7 +++ 8 files changed, 63 insertions(+), 11 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 8715a69f8c..337698e528 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -633,6 +633,10 @@ spec: description: VolumeSize is the underlying cloud volume size format: int32 type: integer + volumeThroughput: + description: Parameter for disks that support provisioned throughput + format: int32 + type: integer volumeType: description: VolumeType is the underlying cloud storage class type: string diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index b31b0c7915..a89a07cbaa 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -527,6 +527,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 6452b33a16..ba93d858c4 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -524,6 +524,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 7a9b122329..88e2761189 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3061,6 +3061,7 @@ func autoConvert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec(in *EtcdMemberSp out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume @@ -3077,6 +3078,7 @@ func autoConvert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec(in *kops.EtcdMem out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 445e512f98..7d91a02480 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1384,6 +1384,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d6dba9dace..2fdb74d3cc 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1534,6 +1534,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go index e2f14afae8..5c28527d1b 100644 --- a/pkg/model/master_volumes.go +++ b/pkg/model/master_volumes.go @@ -37,11 +37,13 @@ import ( ) const ( - DefaultEtcdVolumeSize = 20 - DefaultAWSEtcdVolumeType = "gp2" - DefaultAWSEtcdVolumeIops = 100 - DefaultGCEEtcdVolumeType = "pd-ssd" - DefaultALIEtcdVolumeType = "cloud_ssd" + DefaultEtcdVolumeSize = 20 + DefaultAWSEtcdVolumeType = "gp2" + DefaultAWSEtcdVolumeIops = 100 + DefaultAWSEtcdVolumeGp3Iops = 3000 + DefaultAWSEtcdVolumeGp3Throughput = 125 + DefaultGCEEtcdVolumeType = "pd-ssd" + DefaultALIEtcdVolumeType = "cloud_ssd" ) // MasterVolumeBuilder builds master EBS volumes @@ -118,11 +120,23 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) volumeIops := fi.Int32Value(m.VolumeIops) + volumeThroughput := fi.Int32Value(m.VolumeThroughput) switch volumeType { case "io1": - if volumeIops <= 0 { + if volumeIops <= 100 { volumeIops = DefaultAWSEtcdVolumeIops } + case "io2": + if volumeIops < 100 { + volumeIops = DefaultAWSEtcdVolumeIops + } + case "gp3": + if volumeIops < 3000 { + volumeIops = DefaultAWSEtcdVolumeGp3Iops + } + if volumeThroughput < 125 { + volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput + } default: volumeType = DefaultAWSEtcdVolumeType } @@ -157,12 +171,23 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin Encrypted: fi.Bool(encrypted), Tags: tags, } - if volumeType == "io1" { - t.VolumeIops = i64(int64(volumeIops)) - + if strings.Contains(volumeType, "io") || volumeType == "gp3" { // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html - if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { - return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + t.VolumeIops = i64(int64(volumeIops)) + if volumeType == "io1" { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } else { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 500.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 500. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } + if volumeType == "gp3" { + t.VolumeThroughput = i64(int64(volumeThroughput)) + if float64(*t.VolumeThroughput)/float64(*t.VolumeIops) > 0.25 { + return fmt.Errorf("volumeThroughput to volumeIops ratio must be lower than 0.25. For %s ratio is %f", *t.Name, float64(*t.VolumeThroughput)/float64(*t.VolumeIops)) + } } } diff --git a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go index bddcd6bc53..a45acef7da 100644 --- a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go +++ b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go @@ -40,6 +40,7 @@ type EBSVolume struct { SizeGB *int64 Tags map[string]string VolumeIops *int64 + VolumeThroughput *int64 VolumeType *string } @@ -105,6 +106,7 @@ func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) { Encrypted: v.Encrypted, Name: e.Name, VolumeIops: v.Iops, + VolumeThroughput: v.Throughput, } actual.Tags = mapEC2TagsToMap(v.Tags) @@ -145,6 +147,7 @@ func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) e KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeVolume, e.Tags), } @@ -188,6 +191,7 @@ type terraformVolume struct { Size *int64 `json:"size,omitempty" cty:"size"` Type *string `json:"type,omitempty" cty:"type"` Iops *int64 `json:"iops,omitempty" cty:"iops"` + Throughput *int64 `json:"throughput,omitempty" cty:"throughput"` KmsKeyId *string `json:"kms_key_id,omitempty" cty:"kms_key_id"` Encrypted *bool `json:"encrypted,omitempty" cty:"encrypted"` Tags map[string]string `json:"tags,omitempty" cty:"tags"` @@ -199,6 +203,7 @@ func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: e.Tags, @@ -216,6 +221,7 @@ type cloudformationVolume struct { Size *int64 `json:"Size,omitempty"` Type *string `json:"VolumeType,omitempty"` Iops *int64 `json:"Iops,omitempty"` + Throughput *int64 `json:"Throughput,omitempty"` KmsKeyId *string `json:"KmsKeyId,omitempty"` Encrypted *bool `json:"Encrypted,omitempty"` Tags []cloudformationTag `json:"Tags,omitempty"` @@ -227,6 +233,7 @@ func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget, Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: buildCloudformationTags(e.Tags),