Merge pull request #10453 from msidwell/gp3-etcd

Add gp3 Volume Type to etcd

k8s/crds/kops.k8s.io_clusters.yaml

@@ -825,6 +825,11 @@ spec:
                               size
                             format: int32
                             type: integer
+                          volumeThroughput:
+                            description: Parameter for disks that support provisioned
+                              throughput
+                            format: int32
+                            type: integer
                           volumeType:
                             description: VolumeType is the underlying cloud storage
                               class

pkg/apis/kops/cluster.go

@@ -531,6 +531,8 @@ type EtcdMemberSpec struct {
 	VolumeType *string `json:"volumeType,omitempty"`
 	// If volume type is io1, then we need to specify the number of Iops.
 	VolumeIops *int32 `json:"volumeIops,omitempty"`
+	// Parameter for disks that support provisioned throughput
+	VolumeThroughput *int32 `json:"volumeThroughput,omitempty"`
 	// VolumeSize is the underlying cloud volume size
 	VolumeSize *int32 `json:"volumeSize,omitempty"`
 	// KmsKeyId is a AWS KMS ID used to encrypt the volume

pkg/apis/kops/v1alpha2/cluster.go

@@ -528,6 +528,8 @@ type EtcdMemberSpec struct {
 	VolumeType *string `json:"volumeType,omitempty"`
 	// If volume type is io1, then we need to specify the number of Iops.
 	VolumeIops *int32 `json:"volumeIops,omitempty"`
+	// Parameter for disks that support provisioned throughput
+	VolumeThroughput *int32 `json:"volumeThroughput,omitempty"`
 	// VolumeSize is the underlying cloud volume size
 	VolumeSize *int32 `json:"volumeSize,omitempty"`
 	// KmsKeyId is a AWS KMS ID used to encrypt the volume

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -3213,6 +3213,7 @@ func autoConvert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec(in *EtcdMemberSp
 	out.InstanceGroup = in.InstanceGroup
 	out.VolumeType = in.VolumeType
 	out.VolumeIops = in.VolumeIops
+	out.VolumeThroughput = in.VolumeThroughput
 	out.VolumeSize = in.VolumeSize
 	out.KmsKeyId = in.KmsKeyId
 	out.EncryptedVolume = in.EncryptedVolume
@@ -3229,6 +3230,7 @@ func autoConvert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec(in *kops.EtcdMem
 	out.InstanceGroup = in.InstanceGroup
 	out.VolumeType = in.VolumeType
 	out.VolumeIops = in.VolumeIops
+	out.VolumeThroughput = in.VolumeThroughput
 	out.VolumeSize = in.VolumeSize
 	out.KmsKeyId = in.KmsKeyId
 	out.EncryptedVolume = in.EncryptedVolume

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1482,6 +1482,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.VolumeThroughput != nil {
+		in, out := &in.VolumeThroughput, &out.VolumeThroughput
+		*out = new(int32)
+		**out = **in
+	}
 	if in.VolumeSize != nil {
 		in, out := &in.VolumeSize, &out.VolumeSize
 		*out = new(int32)

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1632,6 +1632,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.VolumeThroughput != nil {
+		in, out := &in.VolumeThroughput, &out.VolumeThroughput
+		*out = new(int32)
+		**out = **in
+	}
 	if in.VolumeSize != nil {
 		in, out := &in.VolumeSize, &out.VolumeSize
 		*out = new(int32)

pkg/model/master_volumes.go

@@ -39,11 +39,13 @@ import (
 )
 
 const (
-	DefaultEtcdVolumeSize    = 20
+	DefaultEtcdVolumeSize             = 20
-	DefaultAWSEtcdVolumeType = "gp2"
+	DefaultAWSEtcdVolumeType          = "gp2"
-	DefaultAWSEtcdVolumeIops = 100
+	DefaultAWSEtcdVolumeIops          = 100
-	DefaultGCEEtcdVolumeType = "pd-ssd"
+	DefaultAWSEtcdVolumeGp3Iops       = 3000
-	DefaultALIEtcdVolumeType = "cloud_ssd"
+	DefaultAWSEtcdVolumeGp3Throughput = 125
+	DefaultGCEEtcdVolumeType          = "pd-ssd"
+	DefaultALIEtcdVolumeType          = "cloud_ssd"
 )
 
 // MasterVolumeBuilder builds master EBS volumes
@@ -122,11 +124,23 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error {
 func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error {
 	volumeType := fi.StringValue(m.VolumeType)
 	volumeIops := fi.Int32Value(m.VolumeIops)
+	volumeThroughput := fi.Int32Value(m.VolumeThroughput)
 	switch volumeType {
 	case "io1":
-		if volumeIops <= 0 {
+		if volumeIops <= 100 {
 			volumeIops = DefaultAWSEtcdVolumeIops
 		}
+	case "io2":
+		if volumeIops < 100 {
+			volumeIops = DefaultAWSEtcdVolumeIops
+		}
+	case "gp3":
+		if volumeIops < 3000 {
+			volumeIops = DefaultAWSEtcdVolumeGp3Iops
+		}
+		if volumeThroughput < 125 {
+			volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput
+		}
 	default:
 		volumeType = DefaultAWSEtcdVolumeType
 	}
@@ -161,12 +175,23 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin
 		Encrypted:        fi.Bool(encrypted),
 		Tags:             tags,
 	}
-	if volumeType == "io1" {
+	if strings.Contains(volumeType, "io") || volumeType == "gp3" {
-		t.VolumeIops = i64(int64(volumeIops))
-
 		// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html
-		if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 {
+		t.VolumeIops = i64(int64(volumeIops))
-			return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+		if volumeType == "io1" {
+			if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 {
+				return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+			}
+		} else {
+			if float64(*t.VolumeIops)/float64(*t.SizeGB) > 500.0 {
+				return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 500. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+			}
+		}
+		if volumeType == "gp3" {
+			t.VolumeThroughput = i64(int64(volumeThroughput))
+			if float64(*t.VolumeThroughput)/float64(*t.VolumeIops) > 0.25 {
+				return fmt.Errorf("volumeThroughput to volumeIops ratio must be lower than 0.25. For %s ratio is %f", *t.Name, float64(*t.VolumeThroughput)/float64(*t.VolumeIops))
+			}
 		}
 	}
 

upup/pkg/fi/cloudup/awstasks/ebsvolume.go

@@ -40,6 +40,7 @@ type EBSVolume struct {
 	SizeGB           *int64
 	Tags             map[string]string
 	VolumeIops       *int64
+	VolumeThroughput *int64
 	VolumeType       *string
 }
 
@@ -105,6 +106,7 @@ func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) {
 		Encrypted:        v.Encrypted,
 		Name:             e.Name,
 		VolumeIops:       v.Iops,
+		VolumeThroughput: v.Throughput,
 	}
 
 	actual.Tags = mapEC2TagsToMap(v.Tags)
@@ -145,6 +147,7 @@ func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) e
 			KmsKeyId:          e.KmsKeyId,
 			Encrypted:         e.Encrypted,
 			Iops:              e.VolumeIops,
+			Throughput:        e.VolumeThroughput,
 			TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeVolume, e.Tags),
 		}
 
@@ -188,6 +191,7 @@ type terraformVolume struct {
 	Size             *int64            `json:"size,omitempty" cty:"size"`
 	Type             *string           `json:"type,omitempty" cty:"type"`
 	Iops             *int64            `json:"iops,omitempty" cty:"iops"`
+	Throughput       *int64            `json:"throughput,omitempty" cty:"throughput"`
 	KmsKeyId         *string           `json:"kms_key_id,omitempty" cty:"kms_key_id"`
 	Encrypted        *bool             `json:"encrypted,omitempty" cty:"encrypted"`
 	Tags             map[string]string `json:"tags,omitempty" cty:"tags"`
@@ -199,6 +203,7 @@ func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes
 		Size:             e.SizeGB,
 		Type:             e.VolumeType,
 		Iops:             e.VolumeIops,
+		Throughput:       e.VolumeThroughput,
 		KmsKeyId:         e.KmsKeyId,
 		Encrypted:        e.Encrypted,
 		Tags:             e.Tags,
@@ -216,6 +221,7 @@ type cloudformationVolume struct {
 	Size             *int64              `json:"Size,omitempty"`
 	Type             *string             `json:"VolumeType,omitempty"`
 	Iops             *int64              `json:"Iops,omitempty"`
+	Throughput       *int64              `json:"Throughput,omitempty"`
 	KmsKeyId         *string             `json:"KmsKeyId,omitempty"`
 	Encrypted        *bool               `json:"Encrypted,omitempty"`
 	Tags             []cloudformationTag `json:"Tags,omitempty"`
@@ -227,6 +233,7 @@ func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget,
 		Size:             e.SizeGB,
 		Type:             e.VolumeType,
 		Iops:             e.VolumeIops,
+		Throughput:       e.VolumeThroughput,
 		KmsKeyId:         e.KmsKeyId,
 		Encrypted:        e.Encrypted,
 		Tags:             buildCloudformationTags(e.Tags),