diff --git a/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml b/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml index f486084510..4ccbcee904 100644 --- a/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml @@ -110,6 +110,12 @@ spec: manager: Karpenter maxSize: 2 minSize: 2 + mixedInstancesPolicy: + instanceRequirements: + cpu: + min: "2" + memory: + min: 2G nodeLabels: kops.k8s.io/instancegroup: nodes role: Node diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 838f78747a..74021a59bc 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -876,6 +876,24 @@ func setupKarpenterNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSub HTTPTokens: fi.String("required"), } + // Karpenter thinks all clusters run VPC CNI and schedules thinking Node Capacity is constrainted by number of ENIs. + + // cpuMin is the reasonable lower limit for a Kubernetes Node + // Generally, it also avoids instances Karpenter thinks it can only schedule 4 Pods on. + cpuMin := resource.MustParse("2") + memoryMin := resource.MustParse(("2G")) + + g.Spec.MixedInstancesPolicy = &api.MixedInstancesPolicySpec{ + InstanceRequirements: &api.InstanceRequirementsSpec{ + CPU: &api.MinMaxSpec{ + Min: &cpuMin, + }, + Memory: &api.MinMaxSpec{ + Min: &memoryMin, + }, + }, + } + return []*api.InstanceGroup{g}, nil } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 491f5fbceb..dc81a56908 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -820,8 +820,9 @@ func karpenterInstanceTypes(cloud awsup.AWSCloud, ig kops.InstanceGroupSpec) ([] hv := ami.VirtualizationType ir := &ec2.InstanceRequirementsRequest{ - VCpuCount: &ec2.VCpuCountRangeRequest{}, - MemoryMiB: &ec2.MemoryMiBRequest{}, + VCpuCount: &ec2.VCpuCountRangeRequest{}, + MemoryMiB: &ec2.MemoryMiBRequest{}, + BurstablePerformance: fi.String("included"), } cpu := instanceRequirements.CPU if cpu != nil { @@ -829,51 +830,58 @@ func karpenterInstanceTypes(cloud awsup.AWSCloud, ig kops.InstanceGroupSpec) ([] cpuMax, _ := instanceRequirements.CPU.Max.AsInt64() ir.VCpuCount.Max = &cpuMax } - if cpu.Min != nil { - cpuMin, _ := instanceRequirements.CPU.Min.AsInt64() - ir.VCpuCount.Min = &cpuMin + cpu := instanceRequirements.CPU + if cpu != nil { + if cpu.Max != nil { + cpuMax, _ := instanceRequirements.CPU.Max.AsInt64() + ir.VCpuCount.Max = &cpuMax + } + if cpu.Min != nil { + cpuMin, _ := instanceRequirements.CPU.Min.AsInt64() + ir.VCpuCount.Min = &cpuMin + } + } else { + ir.VCpuCount.Min = fi.Int64(0) } - } else { - ir.VCpuCount.Min = fi.Int64(0) - } - memory := instanceRequirements.Memory - if memory != nil { - if memory.Max != nil { - memoryMax := instanceRequirements.Memory.Max.ScaledValue(resource.Mega) - ir.MemoryMiB.Max = &memoryMax + memory := instanceRequirements.Memory + if memory != nil { + if memory.Max != nil { + memoryMax := instanceRequirements.Memory.Max.ScaledValue(resource.Mega) + ir.MemoryMiB.Max = &memoryMax + } + if memory.Min != nil { + memoryMin := instanceRequirements.Memory.Min.ScaledValue(resource.Mega) + ir.MemoryMiB.Min = &memoryMin + } + } else { + ir.MemoryMiB.Min = fi.Int64(0) } - if memory.Min != nil { - memoryMin := instanceRequirements.Memory.Min.ScaledValue(resource.Mega) - ir.MemoryMiB.Min = &memoryMin + + ir.AcceleratorCount = &ec2.AcceleratorCountRequest{ + Min: fi.Int64(0), + Max: fi.Int64(0), } - } else { - ir.MemoryMiB.Min = fi.Int64(0) - } - ir.AcceleratorCount = &ec2.AcceleratorCountRequest{ - Min: fi.Int64(0), - Max: fi.Int64(0), + response, err := cloud.EC2().GetInstanceTypesFromInstanceRequirements( + &ec2.GetInstanceTypesFromInstanceRequirementsInput{ + ArchitectureTypes: []*string{arch}, + VirtualizationTypes: []*string{hv}, + InstanceRequirements: ir, + }, + ) + if err != nil { + return nil, err + } + types := []string{} + for _, it := range response.InstanceTypes { + types = append(types, *it.InstanceType) + } + if len(types) == 0 { + return nil, fmt.Errorf("no instances matched requirements") + } + return types, nil } - - response, err := cloud.EC2().GetInstanceTypesFromInstanceRequirements( - &ec2.GetInstanceTypesFromInstanceRequirementsInput{ - ArchitectureTypes: []*string{arch}, - VirtualizationTypes: []*string{hv}, - InstanceRequirements: ir, - }, - ) - if err != nil { - return nil, err - } - types := []string{} - for _, it := range response.InstanceTypes { - types = append(types, *it.InstanceType) - } - if len(types) == 0 { - return nil, fmt.Errorf("no instances matched requirements") - } - return types, nil } } return []string{ig.MachineType}, nil