add gp3 volume default params

add io2 case and correct IOPS minimum value check

add gp3 case

add io2 and gp3 parameter ratio validation logic

add volumeThroughput parameter for disks that support it

add volumeThroughput components throughout ebs structs

add volumeThroughput to versioned api

updated api machinery and crds

apimachinery update

k8s/crds/kops.k8s.io_clusters.yaml

@@ -633,6 +633,10 @@ spec:
                             description: VolumeSize is the underlying cloud volume size
                             format: int32
                             type: integer
+                          volumeThroughput:
+                            description: Parameter for disks that support provisioned throughput
+                            format: int32
+                            type: integer
                           volumeType:
                             description: VolumeType is the underlying cloud storage class
                             type: string

pkg/apis/kops/cluster.go

@@ -527,6 +527,8 @@ type EtcdMemberSpec struct {
 	VolumeType *string `json:"volumeType,omitempty"`
 	// If volume type is io1, then we need to specify the number of Iops.
 	VolumeIops *int32 `json:"volumeIops,omitempty"`
+	// Parameter for disks that support provisioned throughput
+	VolumeThroughput *int32 `json:"volumeThroughput,omitempty"`
 	// VolumeSize is the underlying cloud volume size
 	VolumeSize *int32 `json:"volumeSize,omitempty"`
 	// KmsKeyId is a AWS KMS ID used to encrypt the volume

pkg/apis/kops/v1alpha2/cluster.go

@@ -524,6 +524,8 @@ type EtcdMemberSpec struct {
 	VolumeType *string `json:"volumeType,omitempty"`
 	// If volume type is io1, then we need to specify the number of Iops.
 	VolumeIops *int32 `json:"volumeIops,omitempty"`
+	// Parameter for disks that support provisioned throughput
+	VolumeThroughput *int32 `json:"volumeThroughput,omitempty"`
 	// VolumeSize is the underlying cloud volume size
 	VolumeSize *int32 `json:"volumeSize,omitempty"`
 	// KmsKeyId is a AWS KMS ID used to encrypt the volume

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -3061,6 +3061,7 @@ func autoConvert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec(in *EtcdMemberSp
 	out.InstanceGroup = in.InstanceGroup
 	out.VolumeType = in.VolumeType
 	out.VolumeIops = in.VolumeIops
+	out.VolumeThroughput = in.VolumeThroughput
 	out.VolumeSize = in.VolumeSize
 	out.KmsKeyId = in.KmsKeyId
 	out.EncryptedVolume = in.EncryptedVolume
@@ -3077,6 +3078,7 @@ func autoConvert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec(in *kops.EtcdMem
 	out.InstanceGroup = in.InstanceGroup
 	out.VolumeType = in.VolumeType
 	out.VolumeIops = in.VolumeIops
+	out.VolumeThroughput = in.VolumeThroughput
 	out.VolumeSize = in.VolumeSize
 	out.KmsKeyId = in.KmsKeyId
 	out.EncryptedVolume = in.EncryptedVolume

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1384,6 +1384,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.VolumeThroughput != nil {
+		in, out := &in.VolumeThroughput, &out.VolumeThroughput
+		*out = new(int32)
+		**out = **in
+	}
 	if in.VolumeSize != nil {
 		in, out := &in.VolumeSize, &out.VolumeSize
 		*out = new(int32)

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1534,6 +1534,11 @@ func (in *EtcdMemberSpec) DeepCopyInto(out *EtcdMemberSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.VolumeThroughput != nil {
+		in, out := &in.VolumeThroughput, &out.VolumeThroughput
+		*out = new(int32)
+		**out = **in
+	}
 	if in.VolumeSize != nil {
 		in, out := &in.VolumeSize, &out.VolumeSize
 		*out = new(int32)

pkg/model/master_volumes.go

@@ -37,11 +37,13 @@ import (
 )
 
 const (
-	DefaultEtcdVolumeSize    = 20
+	DefaultEtcdVolumeSize             = 20
-	DefaultAWSEtcdVolumeType = "gp2"
+	DefaultAWSEtcdVolumeType          = "gp2"
-	DefaultAWSEtcdVolumeIops = 100
+	DefaultAWSEtcdVolumeIops          = 100
-	DefaultGCEEtcdVolumeType = "pd-ssd"
+	DefaultAWSEtcdVolumeGp3Iops       = 3000
-	DefaultALIEtcdVolumeType = "cloud_ssd"
+	DefaultAWSEtcdVolumeGp3Throughput = 125
+	DefaultGCEEtcdVolumeType          = "pd-ssd"
+	DefaultALIEtcdVolumeType          = "cloud_ssd"
 )
 
 // MasterVolumeBuilder builds master EBS volumes
@@ -118,11 +120,23 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error {
 func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd kops.EtcdClusterSpec, m kops.EtcdMemberSpec, allMembers []string) error {
 	volumeType := fi.StringValue(m.VolumeType)
 	volumeIops := fi.Int32Value(m.VolumeIops)
+	volumeThroughput := fi.Int32Value(m.VolumeThroughput)
 	switch volumeType {
 	case "io1":
-		if volumeIops <= 0 {
+		if volumeIops <= 100 {
 			volumeIops = DefaultAWSEtcdVolumeIops
 		}
+	case "io2":
+		if volumeIops < 100 {
+			volumeIops = DefaultAWSEtcdVolumeIops
+		}
+	case "gp3":
+		if volumeIops < 3000 {
+			volumeIops = DefaultAWSEtcdVolumeGp3Iops
+		}
+		if volumeThroughput < 125 {
+			volumeThroughput = DefaultAWSEtcdVolumeGp3Throughput
+		}
 	default:
 		volumeType = DefaultAWSEtcdVolumeType
 	}
@@ -157,12 +171,23 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin
 		Encrypted:        fi.Bool(encrypted),
 		Tags:             tags,
 	}
-	if volumeType == "io1" {
+	if strings.Contains(volumeType, "io") || volumeType == "gp3" {
-		t.VolumeIops = i64(int64(volumeIops))
-
 		// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html
-		if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 {
+		t.VolumeIops = i64(int64(volumeIops))
-			return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+		if volumeType == "io1" {
+			if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 {
+				return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+			}
+		} else {
+			if float64(*t.VolumeIops)/float64(*t.SizeGB) > 500.0 {
+				return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 500. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB))
+			}
+		}
+		if volumeType == "gp3" {
+			t.VolumeThroughput = i64(int64(volumeThroughput))
+			if float64(*t.VolumeThroughput)/float64(*t.VolumeIops) > 0.25 {
+				return fmt.Errorf("volumeThroughput to volumeIops ratio must be lower than 0.25. For %s ratio is %f", *t.Name, float64(*t.VolumeThroughput)/float64(*t.VolumeIops))
+			}
 		}
 	}
 

upup/pkg/fi/cloudup/awstasks/ebsvolume.go

@@ -40,6 +40,7 @@ type EBSVolume struct {
 	SizeGB           *int64
 	Tags             map[string]string
 	VolumeIops       *int64
+	VolumeThroughput *int64
 	VolumeType       *string
 }
 
@@ -105,6 +106,7 @@ func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) {
 		Encrypted:        v.Encrypted,
 		Name:             e.Name,
 		VolumeIops:       v.Iops,
+		VolumeThroughput: v.Throughput,
 	}
 
 	actual.Tags = mapEC2TagsToMap(v.Tags)
@@ -145,6 +147,7 @@ func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) e
 			KmsKeyId:          e.KmsKeyId,
 			Encrypted:         e.Encrypted,
 			Iops:              e.VolumeIops,
+			Throughput:        e.VolumeThroughput,
 			TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeVolume, e.Tags),
 		}
 
@@ -188,6 +191,7 @@ type terraformVolume struct {
 	Size             *int64            `json:"size,omitempty" cty:"size"`
 	Type             *string           `json:"type,omitempty" cty:"type"`
 	Iops             *int64            `json:"iops,omitempty" cty:"iops"`
+	Throughput       *int64            `json:"throughput,omitempty" cty:"throughput"`
 	KmsKeyId         *string           `json:"kms_key_id,omitempty" cty:"kms_key_id"`
 	Encrypted        *bool             `json:"encrypted,omitempty" cty:"encrypted"`
 	Tags             map[string]string `json:"tags,omitempty" cty:"tags"`
@@ -199,6 +203,7 @@ func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes
 		Size:             e.SizeGB,
 		Type:             e.VolumeType,
 		Iops:             e.VolumeIops,
+		Throughput:       e.VolumeThroughput,
 		KmsKeyId:         e.KmsKeyId,
 		Encrypted:        e.Encrypted,
 		Tags:             e.Tags,
@@ -216,6 +221,7 @@ type cloudformationVolume struct {
 	Size             *int64              `json:"Size,omitempty"`
 	Type             *string             `json:"VolumeType,omitempty"`
 	Iops             *int64              `json:"Iops,omitempty"`
+	Throughput       *int64              `json:"Throughput,omitempty"`
 	KmsKeyId         *string             `json:"KmsKeyId,omitempty"`
 	Encrypted        *bool               `json:"Encrypted,omitempty"`
 	Tags             []cloudformationTag `json:"Tags,omitempty"`
@@ -227,6 +233,7 @@ func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget,
 		Size:             e.SizeGB,
 		Type:             e.VolumeType,
 		Iops:             e.VolumeIops,
+		Throughput:       e.VolumeThroughput,
 		KmsKeyId:         e.KmsKeyId,
 		Encrypted:        e.Encrypted,
 		Tags:             buildCloudformationTags(e.Tags),