diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 8715a69f8c..337698e528 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -633,6 +633,10 @@ spec: description: VolumeSize is the underlying cloud volume size format: int32 type: integer + volumeThroughput: + description: Parameter for disks that support provisioned throughput + format: int32 + type: integer volumeType: description: VolumeType is the underlying cloud storage class type: string diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index b31b0c7915..a89a07cbaa 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -527,6 +527,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 6452b33a16..ba93d858c4 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -524,6 +524,8 @@ type EtcdMemberSpec struct { VolumeType *string `json:"volumeType,omitempty"` // If volume type is io1, then we need to specify the number of Iops. VolumeIops *int32 `json:"volumeIops,omitempty"` + // Parameter for disks that support provisioned throughput + VolumeThroughput *int32 `json:"volumeThroughput,omitempty"` // VolumeSize is the underlying cloud volume size VolumeSize *int32 `json:"volumeSize,omitempty"` // KmsKeyId is a AWS KMS ID used to encrypt the volume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 7a9b122329..88e2761189 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3061,6 +3061,7 @@ func autoConvert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec(in *EtcdMemberSp out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume @@ -3077,6 +3078,7 @@ func autoConvert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec(in *kops.EtcdMem out.InstanceGroup = in.InstanceGroup out.VolumeType = in.VolumeType out.VolumeIops = in.VolumeIops + out.VolumeThroughput = in.VolumeThroughput out.VolumeSize = in.VolumeSize out.KmsKeyId = in.KmsKeyId out.EncryptedVolume = in.EncryptedVolume diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 445e512f98..7d91a02480 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1384,6 +1384,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d6dba9dace..2fdb74d3cc 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1534,6 +1534,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) { *out = new(int32) **out = **in } + if in.VolumeThroughput != nil { + in, out := &in.VolumeThroughput, &out.VolumeThroughput + *out = new(int32) + **out = **in + } if in.VolumeSize != nil { in, out := &in.VolumeSize, &out.VolumeSize *out = new(int32) diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go index e2f14afae8..5c28527d1b 100644 --- a/pkg/model/master_volumes.go +++ b/pkg/model/master_volumes.go @@ -37,11 +37,13 @@ import ( ) const ( - DefaultEtcdVolumeSize = 20 - DefaultAWSEtcdVolumeType = "gp2" - DefaultAWSEtcdVolumeIops = 100 - DefaultGCEEtcdVolumeType = "pd-ssd" - DefaultALIEtcdVolumeType = "cloud_ssd" + DefaultEtcdVolumeSize = 20 + DefaultAWSEtcdVolumeType = "gp2" + DefaultAWSEtcdVolumeIops = 100 + DefaultAWSEtcdVolumeGp3Iops = 3000 + DefaultAWSEtcdVolumeGp3Throughput = 125 + DefaultGCEEtcdVolumeType = "pd-ssd" + DefaultALIEtcdVolumeType = "cloud_ssd" ) // MasterVolumeBuilder builds master EBS volumes @@ -118,11 +120,23 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) volumeIops := fi.Int32Value(m.VolumeIops) + volumeThroughput := fi.Int32Value(m.VolumeThroughput) switch volumeType { case "io1": - if volumeIops <= 0 { + if volumeIops <= 100 { volumeIops = DefaultAWSEtcdVolumeIops } + case "io2": + if volumeIops < 100 { + volumeIops = DefaultAWSEtcdVolumeIops + } + case "gp3": + if volumeIops < 3000 { + volumeIops = DefaultAWSEtcdVolumeGp3Iops + } + if volumeThroughput < 125 { + volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput + } default: volumeType = DefaultAWSEtcdVolumeType } @@ -157,12 +171,23 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin Encrypted: fi.Bool(encrypted), Tags: tags, } - if volumeType == "io1" { - t.VolumeIops = i64(int64(volumeIops)) - + if strings.Contains(volumeType, "io") || volumeType == "gp3" { // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html - if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { - return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + t.VolumeIops = i64(int64(volumeIops)) + if volumeType == "io1" { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } else { + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 500.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 500. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } + } + if volumeType == "gp3" { + t.VolumeThroughput = i64(int64(volumeThroughput)) + if float64(*t.VolumeThroughput)/float64(*t.VolumeIops) > 0.25 { + return fmt.Errorf("volumeThroughput to volumeIops ratio must be lower than 0.25. For %s ratio is %f", *t.Name, float64(*t.VolumeThroughput)/float64(*t.VolumeIops)) + } } } diff --git a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go index bddcd6bc53..a45acef7da 100644 --- a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go +++ b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go @@ -40,6 +40,7 @@ type EBSVolume struct { SizeGB *int64 Tags map[string]string VolumeIops *int64 + VolumeThroughput *int64 VolumeType *string } @@ -105,6 +106,7 @@ func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) { Encrypted: v.Encrypted, Name: e.Name, VolumeIops: v.Iops, + VolumeThroughput: v.Throughput, } actual.Tags = mapEC2TagsToMap(v.Tags) @@ -145,6 +147,7 @@ func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) e KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeVolume, e.Tags), } @@ -188,6 +191,7 @@ type terraformVolume struct { Size *int64 `json:"size,omitempty" cty:"size"` Type *string `json:"type,omitempty" cty:"type"` Iops *int64 `json:"iops,omitempty" cty:"iops"` + Throughput *int64 `json:"throughput,omitempty" cty:"throughput"` KmsKeyId *string `json:"kms_key_id,omitempty" cty:"kms_key_id"` Encrypted *bool `json:"encrypted,omitempty" cty:"encrypted"` Tags map[string]string `json:"tags,omitempty" cty:"tags"` @@ -199,6 +203,7 @@ func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: e.Tags, @@ -216,6 +221,7 @@ type cloudformationVolume struct { Size *int64 `json:"Size,omitempty"` Type *string `json:"VolumeType,omitempty"` Iops *int64 `json:"Iops,omitempty"` + Throughput *int64 `json:"Throughput,omitempty"` KmsKeyId *string `json:"KmsKeyId,omitempty"` Encrypted *bool `json:"Encrypted,omitempty"` Tags []cloudformationTag `json:"Tags,omitempty"` @@ -227,6 +233,7 @@ func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget, Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, + Throughput: e.VolumeThroughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: buildCloudformationTags(e.Tags),