diff --git a/docs/instance_groups.md b/docs/instance_groups.md index 07efed2f9d..2c6e05ebb5 100644 --- a/docs/instance_groups.md +++ b/docs/instance_groups.md @@ -239,6 +239,25 @@ https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistributi Used only when the Spot allocation strategy is lowest-price. The number of Spot Instance pools across which to allocate your Spot Instances. The Spot pools are determined from the different instance types in the Overrides array of LaunchTemplate. Default if not set is 2. +### instanceRequirements + +{{ kops_feature_table(kops_added_default='1.24') }} + +Instead of configuring specific machine types, the InstanceGroup can be configured to use all machine types that satisfy a given set of requirements. + +``` +spec: + mixedInstancesPolicy: + instanceRquirements: + cpu: + min: "2" + max: "16" + memory: + min: "2G" +``` + +Note that burstable instances are always included in the set of eligible instances. + ## warmPool (AWS Only) {{ kops_feature_table(kops_added_default='1.21') }} diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 46166b3541..39821af52d 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -680,6 +680,41 @@ spec: description: MixedInstancesPolicy defined a optional backing of an AWS ASG by a EC2 Fleet (AWS Only) properties: + instanceRequirements: + description: InstanceRequirements is a list of requirements for + any instance type we are willing to run in the EC2 fleet. + properties: + cpu: + properties: + max: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + min: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + memory: + properties: + max: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + min: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object instances: description: Instances is a list of instance types which we are willing to run in the EC2 fleet diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 5332ea3f30..31a931cc9a 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -17,6 +17,7 @@ limitations under the License. package kops import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -224,6 +225,8 @@ type InstanceMetadataOptions struct { type MixedInstancesPolicySpec struct { // Instances is a list of instance types which we are willing to run in the EC2 fleet Instances []string `json:"instances,omitempty"` + // InstanceRequirements is a list of requirements for any instance type we are willing to run in the EC2 fleet. + InstanceRequirements *InstanceRequirementsSpec `json:"instanceRequirements,omitempty"` // OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"` // OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be @@ -243,6 +246,17 @@ type MixedInstancesPolicySpec struct { SpotInstancePools *int64 `json:"spotInstancePools,omitempty"` } +// InstanceRequirementsSpec is a list of requirements for any instance type we are willing to run in the EC2 fleet. +type InstanceRequirementsSpec struct { + CPU *MinMaxSpec `json:"cpu,omitempty"` + Memory *MinMaxSpec `json:"memory,omitempty"` +} + +type MinMaxSpec struct { + Max *resource.Quantity `json:"max,omitempty"` + Min *resource.Quantity `json:"min,omitempty"` +} + // UserData defines a user-data section type UserData struct { // Name is the name of the user-data diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 8fd13ad656..f6022aef49 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha2 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -171,6 +172,8 @@ type InstanceMetadataOptions struct { type MixedInstancesPolicySpec struct { // Instances is a list of instance types which we are willing to run in the EC2 fleet Instances []string `json:"instances,omitempty"` + // InstanceRequirements is a list of requirements for any instance type we are willing to run in the EC2 fleet. + InstanceRequirements *InstanceRequirementsSpec `json:"instanceRequirements,omitempty"` // OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"` // OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be @@ -190,6 +193,17 @@ type MixedInstancesPolicySpec struct { SpotInstancePools *int64 `json:"spotInstancePools,omitempty"` } +// InstanceRequirementsSpec is a list of requirements for any instance type we are willing to run in the EC2 fleet. +type InstanceRequirementsSpec struct { + CPU *MinMaxSpec `json:"cpu,omitempty"` + Memory *MinMaxSpec `json:"memory,omitempty"` +} + +type MinMaxSpec struct { + Max *resource.Quantity `json:"max,omitempty"` + Min *resource.Quantity `json:"min,omitempty"` +} + // UserData defines a user-data section type UserData struct { // Name is the name of the user-data diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 34c9825afb..40d94197e2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -584,6 +584,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*InstanceRequirementsSpec)(nil), (*kops.InstanceRequirementsSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(a.(*InstanceRequirementsSpec), b.(*kops.InstanceRequirementsSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.InstanceRequirementsSpec)(nil), (*InstanceRequirementsSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec(a.(*kops.InstanceRequirementsSpec), b.(*InstanceRequirementsSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KarpenterConfig)(nil), (*kops.KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(a.(*KarpenterConfig), b.(*kops.KarpenterConfig), scope) }); err != nil { @@ -794,6 +804,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*MinMaxSpec)(nil), (*kops.MinMaxSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(a.(*MinMaxSpec), b.(*kops.MinMaxSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.MinMaxSpec)(nil), (*MinMaxSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(a.(*kops.MinMaxSpec), b.(*MinMaxSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope) }); err != nil { @@ -4543,6 +4563,60 @@ func Convert_kops_InstanceMetadataOptions_To_v1alpha2_InstanceMetadataOptions(in return autoConvert_kops_InstanceMetadataOptions_To_v1alpha2_InstanceMetadataOptions(in, out, s) } +func autoConvert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in *InstanceRequirementsSpec, out *kops.InstanceRequirementsSpec, s conversion.Scope) error { + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(kops.MinMaxSpec) + if err := Convert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.CPU = nil + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(kops.MinMaxSpec) + if err := Convert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Memory = nil + } + return nil +} + +// Convert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec is an autogenerated conversion function. +func Convert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in *InstanceRequirementsSpec, out *kops.InstanceRequirementsSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in, out, s) +} + +func autoConvert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec(in *kops.InstanceRequirementsSpec, out *InstanceRequirementsSpec, s conversion.Scope) error { + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(MinMaxSpec) + if err := Convert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.CPU = nil + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(MinMaxSpec) + if err := Convert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Memory = nil + } + return nil +} + +// Convert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec is an autogenerated conversion function. +func Convert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec(in *kops.InstanceRequirementsSpec, out *InstanceRequirementsSpec, s conversion.Scope) error { + return autoConvert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec(in, out, s) +} + func autoConvert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { out.Enabled = in.Enabled return nil @@ -5729,8 +5803,39 @@ func Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in *kops.M return autoConvert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in, out, s) } +func autoConvert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(in *MinMaxSpec, out *kops.MinMaxSpec, s conversion.Scope) error { + out.Max = in.Max + out.Min = in.Min + return nil +} + +// Convert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec is an autogenerated conversion function. +func Convert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(in *MinMaxSpec, out *kops.MinMaxSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_MinMaxSpec_To_kops_MinMaxSpec(in, out, s) +} + +func autoConvert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(in *kops.MinMaxSpec, out *MinMaxSpec, s conversion.Scope) error { + out.Max = in.Max + out.Min = in.Min + return nil +} + +// Convert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec is an autogenerated conversion function. +func Convert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(in *kops.MinMaxSpec, out *MinMaxSpec, s conversion.Scope) error { + return autoConvert_kops_MinMaxSpec_To_v1alpha2_MinMaxSpec(in, out, s) +} + func autoConvert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error { out.Instances = in.Instances + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(kops.InstanceRequirementsSpec) + if err := Convert_v1alpha2_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(*in, *out, s); err != nil { + return err + } + } else { + out.InstanceRequirements = nil + } out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy out.OnDemandBase = in.OnDemandBase out.OnDemandAboveBase = in.OnDemandAboveBase @@ -5746,6 +5851,15 @@ func Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec( func autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error { out.Instances = in.Instances + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(InstanceRequirementsSpec) + if err := Convert_kops_InstanceRequirementsSpec_To_v1alpha2_InstanceRequirementsSpec(*in, *out, s); err != nil { + return err + } + } else { + out.InstanceRequirements = nil + } out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy out.OnDemandBase = in.OnDemandBase out.OnDemandAboveBase = in.OnDemandAboveBase diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 188d384cd2..f193e59192 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2495,6 +2495,32 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceRequirementsSpec) DeepCopyInto(out *InstanceRequirementsSpec) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceRequirementsSpec. +func (in *InstanceRequirementsSpec) DeepCopy() *InstanceRequirementsSpec { + if in == nil { + return nil + } + out := new(InstanceRequirementsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { *out = *in @@ -3979,6 +4005,32 @@ func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinMaxSpec) DeepCopyInto(out *MinMaxSpec) { + *out = *in + if in.Max != nil { + in, out := &in.Max, &out.Max + x := (*in).DeepCopy() + *out = &x + } + if in.Min != nil { + in, out := &in.Min, &out.Min + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinMaxSpec. +func (in *MinMaxSpec) DeepCopy() *MinMaxSpec { + if in == nil { + return nil + } + out := new(MinMaxSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) { *out = *in @@ -3987,6 +4039,11 @@ func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) *out = make([]string, len(*in)) copy(*out, *in) } + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(InstanceRequirementsSpec) + (*in).DeepCopyInto(*out) + } if in.OnDemandAllocationStrategy != nil { in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy *out = new(string) diff --git a/pkg/apis/kops/v1alpha3/instancegroup.go b/pkg/apis/kops/v1alpha3/instancegroup.go index cf47cb73e4..0b08fb6649 100644 --- a/pkg/apis/kops/v1alpha3/instancegroup.go +++ b/pkg/apis/kops/v1alpha3/instancegroup.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha3 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -168,6 +169,8 @@ type InstanceMetadataOptions struct { type MixedInstancesPolicySpec struct { // Instances is a list of instance types which we are willing to run in the EC2 fleet Instances []string `json:"instances,omitempty"` + // InstanceRequirements is a list of requirements for any instance type we are willing to run in the EC2 fleet. + InstanceRequirements *InstanceRequirementsSpec `json:"instanceRequirements,omitempty"` // OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"` // OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be @@ -187,6 +190,17 @@ type MixedInstancesPolicySpec struct { SpotInstancePools *int64 `json:"spotInstancePools,omitempty"` } +// InstanceRequirementsSpec is a list of requirements for any instance type we are willing to run in the EC2 fleet. +type InstanceRequirementsSpec struct { + CPU *MinMaxSpec `json:"cpu,omitempty"` + Memory *MinMaxSpec `json:"memory,omitempty"` +} + +type MinMaxSpec struct { + Max *resource.Quantity `json:"max,omitempty"` + Min *resource.Quantity `json:"min,omitempty"` +} + // UserData defines a user-data section type UserData struct { // Name is the name of the user-data diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 8f8b85f271..698d757793 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -614,6 +614,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*InstanceRequirementsSpec)(nil), (*kops.InstanceRequirementsSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(a.(*InstanceRequirementsSpec), b.(*kops.InstanceRequirementsSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.InstanceRequirementsSpec)(nil), (*InstanceRequirementsSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec(a.(*kops.InstanceRequirementsSpec), b.(*InstanceRequirementsSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KarpenterConfig)(nil), (*kops.KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(a.(*KarpenterConfig), b.(*kops.KarpenterConfig), scope) }); err != nil { @@ -814,6 +824,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*MinMaxSpec)(nil), (*kops.MinMaxSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(a.(*MinMaxSpec), b.(*kops.MinMaxSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.MinMaxSpec)(nil), (*MinMaxSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(a.(*kops.MinMaxSpec), b.(*MinMaxSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope) }); err != nil { @@ -4433,6 +4453,60 @@ func Convert_kops_InstanceMetadataOptions_To_v1alpha3_InstanceMetadataOptions(in return autoConvert_kops_InstanceMetadataOptions_To_v1alpha3_InstanceMetadataOptions(in, out, s) } +func autoConvert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in *InstanceRequirementsSpec, out *kops.InstanceRequirementsSpec, s conversion.Scope) error { + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(kops.MinMaxSpec) + if err := Convert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.CPU = nil + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(kops.MinMaxSpec) + if err := Convert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Memory = nil + } + return nil +} + +// Convert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec is an autogenerated conversion function. +func Convert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in *InstanceRequirementsSpec, out *kops.InstanceRequirementsSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(in, out, s) +} + +func autoConvert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec(in *kops.InstanceRequirementsSpec, out *InstanceRequirementsSpec, s conversion.Scope) error { + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(MinMaxSpec) + if err := Convert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.CPU = nil + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(MinMaxSpec) + if err := Convert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Memory = nil + } + return nil +} + +// Convert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec is an autogenerated conversion function. +func Convert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec(in *kops.InstanceRequirementsSpec, out *InstanceRequirementsSpec, s conversion.Scope) error { + return autoConvert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec(in, out, s) +} + func autoConvert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { out.Enabled = in.Enabled return nil @@ -5597,8 +5671,39 @@ func Convert_kops_MetricsServerConfig_To_v1alpha3_MetricsServerConfig(in *kops.M return autoConvert_kops_MetricsServerConfig_To_v1alpha3_MetricsServerConfig(in, out, s) } +func autoConvert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(in *MinMaxSpec, out *kops.MinMaxSpec, s conversion.Scope) error { + out.Max = in.Max + out.Min = in.Min + return nil +} + +// Convert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec is an autogenerated conversion function. +func Convert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(in *MinMaxSpec, out *kops.MinMaxSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_MinMaxSpec_To_kops_MinMaxSpec(in, out, s) +} + +func autoConvert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(in *kops.MinMaxSpec, out *MinMaxSpec, s conversion.Scope) error { + out.Max = in.Max + out.Min = in.Min + return nil +} + +// Convert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec is an autogenerated conversion function. +func Convert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(in *kops.MinMaxSpec, out *MinMaxSpec, s conversion.Scope) error { + return autoConvert_kops_MinMaxSpec_To_v1alpha3_MinMaxSpec(in, out, s) +} + func autoConvert_v1alpha3_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error { out.Instances = in.Instances + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(kops.InstanceRequirementsSpec) + if err := Convert_v1alpha3_InstanceRequirementsSpec_To_kops_InstanceRequirementsSpec(*in, *out, s); err != nil { + return err + } + } else { + out.InstanceRequirements = nil + } out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy out.OnDemandBase = in.OnDemandBase out.OnDemandAboveBase = in.OnDemandAboveBase @@ -5614,6 +5719,15 @@ func Convert_v1alpha3_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec( func autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha3_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error { out.Instances = in.Instances + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(InstanceRequirementsSpec) + if err := Convert_kops_InstanceRequirementsSpec_To_v1alpha3_InstanceRequirementsSpec(*in, *out, s); err != nil { + return err + } + } else { + out.InstanceRequirements = nil + } out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy out.OnDemandBase = in.OnDemandBase out.OnDemandAboveBase = in.OnDemandAboveBase diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 8ef416d7cb..435862bb7c 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -2401,6 +2401,32 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceRequirementsSpec) DeepCopyInto(out *InstanceRequirementsSpec) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceRequirementsSpec. +func (in *InstanceRequirementsSpec) DeepCopy() *InstanceRequirementsSpec { + if in == nil { + return nil + } + out := new(InstanceRequirementsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { *out = *in @@ -3862,6 +3888,32 @@ func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinMaxSpec) DeepCopyInto(out *MinMaxSpec) { + *out = *in + if in.Max != nil { + in, out := &in.Max, &out.Max + x := (*in).DeepCopy() + *out = &x + } + if in.Min != nil { + in, out := &in.Min, &out.Min + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinMaxSpec. +func (in *MinMaxSpec) DeepCopy() *MinMaxSpec { + if in == nil { + return nil + } + out := new(MinMaxSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) { *out = *in @@ -3870,6 +3922,11 @@ func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) *out = make([]string, len(*in)) copy(*out, *in) } + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(InstanceRequirementsSpec) + (*in).DeepCopyInto(*out) + } if in.OnDemandAllocationStrategy != nil { in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy *out = new(string) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 74dc6579f0..acdf974912 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2559,6 +2559,32 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceRequirementsSpec) DeepCopyInto(out *InstanceRequirementsSpec) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(MinMaxSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceRequirementsSpec. +func (in *InstanceRequirementsSpec) DeepCopy() *InstanceRequirementsSpec { + if in == nil { + return nil + } + out := new(InstanceRequirementsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { *out = *in @@ -4075,6 +4101,32 @@ func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinMaxSpec) DeepCopyInto(out *MinMaxSpec) { + *out = *in + if in.Max != nil { + in, out := &in.Max, &out.Max + x := (*in).DeepCopy() + *out = &x + } + if in.Min != nil { + in, out := &in.Min, &out.Min + x := (*in).DeepCopy() + *out = &x + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinMaxSpec. +func (in *MinMaxSpec) DeepCopy() *MinMaxSpec { + if in == nil { + return nil + } + out := new(MinMaxSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) { *out = *in @@ -4083,6 +4135,11 @@ func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) *out = make([]string, len(*in)) copy(*out, *in) } + if in.InstanceRequirements != nil { + in, out := &in.InstanceRequirements, &out.InstanceRequirements + *out = new(InstanceRequirementsSpec) + (*in).DeepCopyInto(*out) + } if in.OnDemandAllocationStrategy != nil { in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy *out = new(string) diff --git a/pkg/model/awsmodel/BUILD.bazel b/pkg/model/awsmodel/BUILD.bazel index bfe6210998..06360d4a59 100644 --- a/pkg/model/awsmodel/BUILD.bazel +++ b/pkg/model/awsmodel/BUILD.bazel @@ -37,6 +37,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/iam:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 6cd960e1d5..4f2acae8aa 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" @@ -495,6 +496,40 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil if ig.Spec.MixedInstancesPolicy != nil { spec := ig.Spec.MixedInstancesPolicy + if spec.InstanceRequirements != nil { + + ir := &awstasks.InstanceRequirements{} + + cpu := spec.InstanceRequirements.CPU + if cpu != nil { + if cpu.Max != nil { + cpuMax, _ := spec.InstanceRequirements.CPU.Max.AsInt64() + ir.CPUMax = &cpuMax + } + if cpu.Min != nil { + cpuMin, _ := spec.InstanceRequirements.CPU.Min.AsInt64() + ir.CPUMin = &cpuMin + } + } else { + ir.CPUMin = fi.Int64(0) + } + + memory := spec.InstanceRequirements.Memory + if memory != nil { + if memory.Max != nil { + memoryMax := spec.InstanceRequirements.Memory.Max.ScaledValue(resource.Mega) + ir.MemoryMax = &memoryMax + } + if memory.Min != nil { + memoryMin := spec.InstanceRequirements.Memory.Min.ScaledValue(resource.Mega) + ir.MemoryMin = &memoryMin + } + } else { + ir.MemoryMin = fi.Int64(0) + } + t.InstanceRequirements = ir + } + t.MixedInstanceOverrides = spec.Instances t.MixedOnDemandAboveBase = spec.OnDemandAboveBase t.MixedOnDemandAllocationStrategy = spec.OnDemandAllocationStrategy diff --git a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel index 0345b14b93..7a4b2ea53a 100644 --- a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel @@ -43,6 +43,7 @@ go_library( "iamrolepolicy_fitask.go", "instance.go", "instance_fitask.go", + "instancerequirements.go", "internetgateway.go", "internetgateway_fitask.go", "launchtemplate.go", diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go index f0b83e9102..c68fa0c086 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "k8s.io/klog/v2" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/cloudformation" @@ -60,6 +61,8 @@ type AutoscalingGroup struct { MinSize *int64 // MixedInstanceOverrides is a collection of instance types to use with fleet policy MixedInstanceOverrides []string + // InstanceRequirements is a list of requirements for any instance type we are willing to run in the EC2 fleet. + InstanceRequirements *InstanceRequirements // MixedOnDemandAllocationStrategy is allocation strategy to use for on-demand instances MixedOnDemandAllocationStrategy *string // MixedOnDemandBase is percentage split of On-Demand Instances and Spot Instances for your @@ -231,6 +234,9 @@ func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) { } } + ir, _ := findInstanceRequirements(g) + actual.InstanceRequirements = ir + if subnetSlicesEqualIgnoreOrder(actual.Subnets, e.Subnets) { actual.Subnets = e.Subnets } @@ -377,6 +383,9 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos }, ) } + if e.InstanceRequirements != nil { + p.Overrides = append(p.Overrides, overridesFromInstanceRequirements(e.InstanceRequirements)) + } } else if e.LaunchTemplate != nil { request.LaunchTemplate = &autoscaling.LaunchTemplateSpecification{ LaunchTemplateId: e.LaunchTemplate.ID, @@ -471,7 +480,7 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos setup(request).InstancesDistribution.SpotMaxPrice = e.MixedSpotMaxPrice changes.MixedSpotMaxPrice = nil } - if changes.MixedInstanceOverrides != nil { + if changes.MixedInstanceOverrides != nil || changes.InstanceRequirements != nil { if setup(request).LaunchTemplate == nil { setup(request).LaunchTemplate = &autoscaling.LaunchTemplate{ LaunchTemplateSpecification: &autoscaling.LaunchTemplateSpecification{ @@ -481,11 +490,20 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos } } - p := request.MixedInstancesPolicy.LaunchTemplate - for _, x := range changes.MixedInstanceOverrides { - p.Overrides = append(p.Overrides, &autoscaling.LaunchTemplateOverrides{InstanceType: fi.String(x)}) + if changes.MixedInstanceOverrides != nil { + p := request.MixedInstancesPolicy.LaunchTemplate + for _, x := range changes.MixedInstanceOverrides { + p.Overrides = append(p.Overrides, &autoscaling.LaunchTemplateOverrides{InstanceType: fi.String(x)}) + } + changes.MixedInstanceOverrides = nil + } + + if changes.InstanceRequirements != nil { + p := request.MixedInstancesPolicy.LaunchTemplate + + p.Overrides = append(p.Overrides, overridesFromInstanceRequirements(changes.InstanceRequirements)) + changes.InstanceRequirements = nil } - changes.MixedInstanceOverrides = nil } if changes.MinSize != nil { @@ -670,6 +688,9 @@ func (e *AutoscalingGroup) UseMixedInstancesPolicy() bool { if e.MixedSpotMaxPrice != nil { return true } + if e.InstanceRequirements != nil { + return true + } return false } diff --git a/upup/pkg/fi/cloudup/awstasks/instancerequirements.go b/upup/pkg/fi/cloudup/awstasks/instancerequirements.go new file mode 100644 index 0000000000..2e8fbc1d7c --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/instancerequirements.go @@ -0,0 +1,73 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "github.com/aws/aws-sdk-go/service/autoscaling" + + "k8s.io/kops/upup/pkg/fi" +) + +type InstanceRequirements struct { + Architecture *string + CPUMin *int64 + CPUMax *int64 + MemoryMin *int64 + MemoryMax *int64 +} + +var _ fi.HasDependencies = &InstanceRequirements{} + +func (e *InstanceRequirements) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +func findInstanceRequirements(asg *autoscaling.Group) (*InstanceRequirements, error) { + actual := &InstanceRequirements{} + if asg.MixedInstancesPolicy != nil { + for _, override := range asg.MixedInstancesPolicy.LaunchTemplate.Overrides { + if override.InstanceRequirements != nil { + if override.InstanceRequirements.VCpuCount != nil { + actual.CPUMax = override.InstanceRequirements.VCpuCount.Max + actual.CPUMin = override.InstanceRequirements.VCpuCount.Min + } + if override.InstanceRequirements.MemoryMiB != nil { + actual.MemoryMax = override.InstanceRequirements.MemoryMiB.Max + actual.MemoryMax = override.InstanceRequirements.MemoryMiB.Min + } + return actual, nil + } + } + } + return nil, nil +} + +func overridesFromInstanceRequirements(ir *InstanceRequirements) *autoscaling.LaunchTemplateOverrides { + return &autoscaling.LaunchTemplateOverrides{ + InstanceRequirements: &autoscaling.InstanceRequirements{ + VCpuCount: &autoscaling.VCpuCountRequest{ + Max: ir.CPUMax, + Min: ir.CPUMin, + }, + MemoryMiB: &autoscaling.MemoryMiBRequest{ + Max: ir.MemoryMax, + Min: ir.MemoryMin, + }, + BurstablePerformance: fi.String("included"), + }, + } +}