kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade Karpenter to v0.27.5 (#15144) 

* feat(karpenter): Upgrade to version 0.27.0

Upgrade Karpenter to current last stable version `0.27.0`.
Template have been updated to use the same templates than the Helm chart.

* feat(karpenter): Use AWSNodeTemplate for launchTemplate

To set Launch Templates is deprecated into the provisioner, it is recommends using the `AWSNodeTemplate` to set it.
Ref:
 - https://karpenter.sh/v0.27.0/concepts/node-templates/

* feat(karpenter): Enable pruning addon

* Use extra flags in upgrade-ab scenario test

* feat(karpenter): Drop `karpenter` feature flag

* feat(karpenter): Add release note for `1.27`

* feat(karpenter): Upgrade to version 0.27.3

* feat(karpenter):  fix template

* feat(karpenter): Upgrade to version 0.27.5

* Update Karpenter documentation with depending kops version

* Delete KOPS_FEATURE_FLAGS from e2e test `run-test`

* Run hack/update-expected.sh
This commit is contained in:
Tone 2023-06-30 07:57:45 +02:00 committed by GitHub
parent 403dc27e25
commit c2ed4b6f64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 816 additions and 709 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/kops/create_cluster.go
View File

@ -474,12 +474,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
return pflag.NormalizedName(name)
})
if featureflag.Karpenter.Enabled() {
cmd.Flags().StringVar(&options.InstanceManager, "instance-manager", options.InstanceManager, "Instance manager to use (cloudgroups or karpenter. Default: cloudgroups)")
cmd.RegisterFlagCompletionFunc("instance-manager", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return []string{"cloudgroups", "karpenter"}, cobra.ShellCompDirectiveNoFileComp
})
}
cmd.Flags().StringVar(&options.InstanceManager, "instance-manager", options.InstanceManager, "Instance manager to use (cloudgroups or karpenter. Default: cloudgroups)")
cmd.RegisterFlagCompletionFunc("instance-manager", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return []string{"cloudgroups", "karpenter"}, cobra.ShellCompDirectiveNoFileComp
})
return cmd
}

5
cmd/kops/create_cluster_integration_test.go
View File

@ -186,11 +186,6 @@ func TestCreateClusterDifferentAMIs(t *testing.T) {
// TestCreateClusterKarpenter runs kops create cluster --instance-manager=karpenter
func TestCreateClusterKarpenter(t *testing.T) {
featureflag.ParseFlags("+Karpenter")
unsetFeatureFlags := func() {
featureflag.ParseFlags("-Karpenter")
}
defer unsetFeatureFlags()
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/karpenter", "v1alpha2")
}

6
cmd/kops/integration_test.go
View File

@ -985,12 +985,6 @@ func TestExternalDNSIRSA(t *testing.T) {
}
func TestKarpenter(t *testing.T) {
featureflag.ParseFlags("+Karpenter")
unsetFeatureFlags := func() {
featureflag.ParseFlags("-Karpenter")
}
defer unsetFeatureFlags()
test := newIntegrationTest("minimal.example.com", "karpenter").
withOIDCDiscovery().
withDefaults24().

1
docs/cli/kops_create_cluster.md generated
View File

@ -95,6 +95,7 @@ kops create cluster [CLUSTER] [flags]
--gce-service-account string Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.
-h, --help help for cluster
--image string Machine image for all instances
--instance-manager string Instance manager to use (cloudgroups or karpenter. Default: cloudgroups) (default "cloudgroups")
--ipv6 Use IPv6 for the pod network (AWS only)
--kubernetes-feature-gates strings List of Kubernetes feature gates to enable/disable
--kubernetes-version string Version of Kubernetes to run (defaults to version in channel)

4
docs/operations/karpenter.md
View File

@ -2,11 +2,9 @@
[Karpenter](https://karpenter.sh) is a Kubernetes-native capacity manager that directly provisions Nodes and underlying instances based on Pod requirements. On AWS, kOps supports managing an InstanceGroup with either Karpenter or an AWS Auto Scaling Group (ASG).
Karpenter is a fairly new project, and it is still not determined how Karpenter should work with kOps. Because of this, Karpenter is behind the `Karpenter` feature flag.
## Installing
Enable the Karpenter feature flag:
If using kOps 1.26 or older, enable the Karpenter feature flag :
```sh
export KOPS_FEATURE_FLAGS="Karpenter"

2
docs/releases/1.27-NOTES.md
View File

@ -13,6 +13,8 @@ This behaviour can be overridden by setting `spec.etcdClusters[*].manager.backup
* external-dns is now supported in IPv6 clusters.
* `Karpenter` has been upgraded to version `0.27.x` and the feature flag is no longer necessary.
## AWS
* As of Kubernetes version 1.27, all nodes will default to running with instance-metadata-service tokens required, with a max hop limit of 1.

4
pkg/apis/kops/validation/validation.go
View File

@ -38,7 +38,6 @@ import (
"k8s.io/kops/pkg/util/subnet"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/pkg/model/components"
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/upup/pkg/fi"
@ -278,9 +277,6 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
if !fi.ValueOf(spec.IAM.UseServiceAccountExternalPermissions) {
allErrs = append(allErrs, field.Forbidden(fldPath, "Karpenter requires that service accounts use external permissions"))
}
if !featureflag.Karpenter.Enabled() {
allErrs = append(allErrs, field.Forbidden(fldPath, "karpenter requires the Karpenter feature flag"))
}
}
if spec.CertManager != nil && fi.ValueOf(spec.CertManager.Enabled) {

2
pkg/featureflag/featureflag.go
View File

@ -81,8 +81,6 @@ var (
UseAddonOperators = new("UseAddonOperators", Bool(false))
// TerraformManagedFiles enables rendering managed files into the Terraform configuration.
TerraformManagedFiles = new("TerraformManagedFiles", Bool(true))
// Karpenter enables karpenter-managed Instance Groups
Karpenter = new("Karpenter", Bool(false))
// ImageDigest remaps all manifests with image digests
ImageDigest = new("ImageDigest", Bool(true))
// Scaleway toggles the Scaleway Cloud support.

4
tests/e2e/kubetest2-kops/deployer/common.go
View File

@ -228,6 +228,10 @@ func (d *deployer) featureFlags() string {
return e[1]
}
}
// if not set by the env flag, but set in the environment, use that.
if e := os.Getenv("KOPS_FEATURE_FLAGS"); e != "" {
return e
}
return ""
}

2
tests/e2e/scenarios/lib/common.sh
View File

@ -55,7 +55,7 @@ if [[ -z "${AWS_SSH_PUBLIC_KEY_FILE-}" ]]; then
fi
KUBETEST2="kubetest2 kops -v=2 --cloud-provider=${CLOUD_PROVIDER} --cluster-name=${CLUSTER_NAME:-} --kops-root=${REPO_ROOT}"
KUBETEST2="${KUBETEST2} --admin-access=${ADMIN_ACCESS:-} --env=KOPS_FEATURE_FLAGS=${KOPS_FEATURE_FLAGS:-}"
KUBETEST2="${KUBETEST2} --admin-access=${ADMIN_ACCESS:-}"
if [[ -n "${GCP_PROJECT-}" ]]; then
KUBETEST2="${KUBETEST2} --gcp-project=${GCP_PROJECT}"

7
tests/e2e/scenarios/upgrade-ab/run-test.sh
View File

@ -78,7 +78,10 @@ else
KOPS="${KOPS_A}"
fi
create_args=""
if [[ ${KOPS_IRSA-} = true ]]; then
create_args="${create_args} --discovery-store=${DISCOVERY_STORE}/${CLUSTER_NAME}/discovery"
fi
${KUBETEST2} \
--up \
@ -86,7 +89,7 @@ ${KUBETEST2} \
--kubernetes-version="${K8S_VERSION_A}" \
--control-plane-size="${KOPS_CONTROL_PLANE_SIZE:-1}" \
--template-path="${KOPS_TEMPLATE:-}" \
--create-args="--networking calico"
--create-args="--networking calico ${KOPS_EXTRA_FLAGS:-} ${create_args}"
# Export kubeconfig-a
KUBECONFIG_A=$(mktemp -t kops.XXXXXXXXX)

54
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
View File

@ -69,8 +69,60 @@ spec:
version: 9.99.0
- id: k8s-1.19
manifest: karpenter.sh/k8s-1.19.yaml
manifestHash: b6cd8f0b7dcdf48fc658eb95d9948900665562de99e317c565fdfbfbdc4481bb
manifestHash: 0f922f2cfc90670293583bd428cd7ca393181fd8057cdd8382cbf247787f0c6c
name: karpenter.sh
prune:
kinds:
- kind: ConfigMap
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- kind: Service
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- kind: ServiceAccount
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- group: admissionregistration.k8s.io
kind: MutatingWebhookConfiguration
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: admissionregistration.k8s.io
kind: ValidatingWebhookConfiguration
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: apps
kind: DaemonSet
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: apps
kind: Deployment
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- group: apps
kind: StatefulSet
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: policy
kind: PodDisruptionBudget
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- group: rbac.authorization.k8s.io
kind: ClusterRole
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: rbac.authorization.k8s.io
kind: ClusterRoleBinding
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
- group: rbac.authorization.k8s.io
kind: Role
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
- group: rbac.authorization.k8s.io
kind: RoleBinding
labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops
namespaces:
- kube-system
selector:
k8s-addon: karpenter.sh
version: 9.99.0

566
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content
View File

@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.8.0
controller-gen.kubebuilder.io/version: v0.11.3
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
@ -44,6 +44,11 @@ spec:
Node properties are determined from a combination of provisioner and
pod scheduling constraints.
properties:
annotations:
additionalProperties:
type: string
description: Annotations are applied to every node.
type: object
consolidation:
description: Consolidation are the consolidation parameters
properties:
@ -65,6 +70,10 @@ spec:
description: ContainerRuntime is the container runtime to be used
with your worker nodes.
type: string
cpuCFSQuota:
description: CPUCFSQuota enables CPU CFS quota enforcement for
containers that specify CPU limits.
type: boolean
evictionHard:
additionalProperties:
type: string
@ -89,6 +98,27 @@ spec:
description: EvictionSoftGracePeriod is the map of signal names
to quantities that define grace periods for each eviction signal
type: object
imageGCHighThresholdPercent:
description: ImageGCHighThresholdPercent is the percent of disk
usage after which image garbage collection is always run. The
percent is calculated by dividing this field value by 100, so
this field must be between 0 and 100, inclusive. When specified,
the value must be greater than ImageGCLowThresholdPercent.
format: int32
maximum: 100
minimum: 0
type: integer
imageGCLowThresholdPercent:
description: ImageGCLowThresholdPercent is the percent of disk
usage before which image garbage collection is never run. Lowest
disk usage to garbage collect to. The percent is calculated
by dividing this field value by 100, so the field value must
be between 0 and 100, inclusive. When specified, the value must
be less than imageGCHighThresholdPercent
format: int32
maximum: 100
minimum: 0
type: integer
kubeReserved:
additionalProperties:
anyOf:
@ -161,6 +191,8 @@ spec:
name:
description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names'
type: string
required:
- name
type: object
requirements:
description: Requirements are layered with Labels and applied to every
@ -343,12 +375,6 @@ spec:
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
---
@ -356,7 +382,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.8.0
controller-gen.kubebuilder.io/version: v0.11.3
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
@ -483,6 +509,10 @@ spec:
context:
description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html
type: string
detailedMonitoring:
description: DetailedMonitoring controls if detailed monitoring is
enabled for instances that are launched
type: boolean
instanceProfile:
description: InstanceProfile is the AWS identity that instances use.
type: string
@ -570,17 +600,43 @@ spec:
being provisioned with the correct configuration.
type: string
type: object
status:
description: AWSNodeTemplateStatus contains the resolved state of the
AWSNodeTemplate
properties:
securityGroups:
description: SecurityGroups contains the current Security Groups values
that are available to the cluster under the SecurityGroups selectors.
items:
description: SecurityGroupStatus contains resolved SecurityGroup
selector values utilized for node launch
properties:
id:
description: Id of the security group
type: string
type: object
type: array
subnets:
description: Subnets contains the current Subnet values that are available
to the cluster under the subnet selectors.
items:
description: SubnetStatus contains resolved Subnet selector values
utilized for node launch
properties:
id:
description: Id of the subnet
type: string
zone:
description: The associated availability zone
type: string
type: object
type: array
type: object
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
---
@ -606,19 +662,18 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook-cert
name: karpenter-cert
namespace: kube-system
---
apiVersion: v1
data:
loglevel.controller: debug
loglevel.webhook: debug
zap-logger-config: |
{
"level": "debug",
"development": true,
"development": false,
"disableStacktrace": true,
"disableCaller": true,
"sampling": {
@ -627,7 +682,7 @@ data:
},
"outputPaths": ["stdout"],
"errorOutputPaths": ["stderr"],
"encoding": "console",
"encoding": "json",
"encoderConfig": {
"timeKey": "time",
"levelKey": "level",
@ -647,13 +702,22 @@ metadata:
app.kubernetes.io/managed-by: kops
app.kubernetes.io/part-of: karpenter
k8s-addon: karpenter.sh
name: karpenter-config-logging
name: config-logging
namespace: kube-system
---
apiVersion: v1
data:
aws.clusterEndpoint: https://api.internal.minimal.example.com
aws.clusterName: minimal.example.com
aws.defaultInstanceProfile: ""
aws.enableENILimitedPodDensity: "true"
aws.enablePodENI: "false"
aws.interruptionQueueName: ""
aws.isolatedVPC: "false"
aws.nodeNameConvention: resource-name
aws.vmMemoryOverheadPercent: "0.075"
batchIdleDuration: 1s
batchMaxDuration: 10s
kind: ConfigMap
@ -662,7 +726,6 @@ metadata:
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
app.kubernetes.io/part-of: karpenter
k8s-addon: karpenter.sh
name: karpenter-global-settings
namespace: kube-system
@ -677,7 +740,7 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-controller
name: karpenter-admin
rules:
- apiGroups:
- karpenter.sh
@ -688,6 +751,9 @@ rules:
- get
- list
- watch
- create
- delete
- patch
- apiGroups:
- karpenter.k8s.aws
resources:
@ -696,15 +762,42 @@ rules:
- get
- list
- watch
- create
- delete
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-core
rules:
- apiGroups:
- karpenter.sh
resources:
- provisioners
- provisioners/status
- machines
- machines/status
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- pods
- nodes
- namespaces
- persistentvolumes
- persistentvolumeclaims
- replicationcontrollers
- namespaces
verbs:
- get
- list
@ -728,6 +821,15 @@ rules:
verbs:
- list
- watch
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
- mutatingwebhookconfigurations
verbs:
- get
- watch
- list
- apiGroups:
- policy
resources:
@ -740,6 +842,8 @@ rules:
- karpenter.sh
resources:
- provisioners/status
- machines
- machines/status
verbs:
- create
- delete
@ -765,6 +869,15 @@ rules:
- pods/eviction
verbs:
- create
- apiGroups:
- admissionregistration.k8s.io
resourceNames:
- validation.webhook.karpenter.sh
- validation.webhook.config.karpenter.sh
resources:
- validatingwebhookconfigurations
verbs:
- update
---
@ -776,22 +889,20 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
name: karpenter
rules:
- apiGroups:
- admissionregistration.k8s.io
- karpenter.k8s.aws
resources:
- validatingwebhookconfigurations
- mutatingwebhookconfigurations
- awsnodetemplates
verbs:
- get
- watch
- list
- watch
- apiGroups:
- admissionregistration.k8s.io
resourceNames:
- validation.webhook.provisioners.karpenter.sh
- validation.webhook.config.karpenter.sh
- validation.webhook.karpenter.k8s.aws
resources:
- validatingwebhookconfigurations
verbs:
@ -799,11 +910,18 @@ rules:
- apiGroups:
- admissionregistration.k8s.io
resourceNames:
- defaulting.webhook.provisioners.karpenter.sh
- defaulting.webhook.karpenter.k8s.aws
resources:
- mutatingwebhookconfigurations
verbs:
- update
- apiGroups:
- karpenter.k8s.aws
resources:
- awsnodetemplates/status
verbs:
- patch
- update
---
@ -815,11 +933,11 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-controller
name: karpenter-core
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: karpenter-controller
name: karpenter-core
subjects:
- kind: ServiceAccount
name: karpenter
@ -835,11 +953,11 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
name: karpenter
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: karpenter-webhook
name: karpenter
subjects:
- kind: ServiceAccount
name: karpenter
@ -855,7 +973,7 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-controller
name: karpenter
namespace: kube-system
rules:
- apiGroups:
@ -878,7 +996,7 @@ rules:
- apiGroups:
- ""
resourceNames:
- karpenter-webhook-cert
- karpenter-cert
resources:
- secrets
verbs:
@ -887,7 +1005,7 @@ rules:
- ""
resourceNames:
- karpenter-global-settings
- karpenter-config-logging
- config-logging
resources:
- configmaps
verbs:
@ -930,35 +1048,17 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
name: karpenter-dns
namespace: kube-system
rules:
- apiGroups:
- ""
resourceNames:
- kube-dns
resources:
- namespaces
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- update
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- watch
- create
- update
---
@ -970,12 +1070,12 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-controller
name: karpenter
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: karpenter-controller
name: karpenter
subjects:
- kind: ServiceAccount
name: karpenter
@ -991,12 +1091,12 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
name: karpenter-dns
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: karpenter-webhook
name: karpenter-dns
subjects:
- kind: ServiceAccount
name: karpenter
@ -1012,33 +1112,21 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-metrics
name: karpenter
namespace: kube-system
spec:
ports:
- port: 8080
- name: http-metrics
port: 8080
protocol: TCP
targetPort: http-metrics
selector:
karpenter: controller
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
namespace: kube-system
spec:
ports:
- port: 443
- name: https-webhook
port: 443
protocol: TCP
targetPort: https-webhook
selector:
karpenter: webhook
type: ClusterIP
---
@ -1050,151 +1138,22 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-controller
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
karpenter: controller
strategy:
type: Recreate
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: karpenter
karpenter: controller
kops.k8s.io/managed-by: kops
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
- key: karpenter.sh/provisioner-name
operator: DoesNotExist
containers:
- env:
- name: AWS_ENI_LIMITED_POD_DENSITY
value: "true"
- name: AWS_NODE_NAME_CONVENTION
value: resource-name
- name: AWS_REGION
value: us-test-1
- name: CLUSTER_NAME
value: minimal.example.com
- name: CLUSTER_ENDPOINT
value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME
value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: MEMORY_LIMIT
valueFrom:
resourceFieldRef:
containerName: controller
divisor: "0"
resource: limits.memory
- name: AWS_ROLE_ARN
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/controller:v0.16.3
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 30
timeoutSeconds: 30
name: controller
ports:
- containerPort: 8080
name: http-metrics
protocol: TCP
- containerPort: 8081
name: http
protocol: TCP
readinessProbe:
httpGet:
path: /readyz
port: http
timeoutSeconds: 30
resources:
limits:
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
volumeMounts:
- mountPath: /var/run/secrets/amazonaws.com/
name: token-amazonaws-com
readOnly: true
dnsPolicy: Default
priorityClassName: system-cluster-critical
securityContext:
fsGroup: 10001
serviceAccountName: karpenter
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
topologySpreadConstraints:
- labelSelector:
matchLabels:
karpenter: webhook
maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
- labelSelector:
matchLabels:
karpenter: webhook
maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
volumes:
- name: token-amazonaws-com
projected:
defaultMode: 420
sources:
- serviceAccountToken:
audience: amazonaws.com
expirationSeconds: 86400
path: token
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-webhook
name: karpenter
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
karpenter: webhook
strategy:
type: Recreate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/instance: karpenter
app.kubernetes.io/name: karpenter
karpenter: webhook
kops.k8s.io/managed-by: kops
@ -1221,58 +1180,73 @@ spec:
operator: DoesNotExist
- key: node-role.kubernetes.io/master
operator: Exists
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app.kubernetes.io/instance
operator: In
values:
- karpenter
topologyKey: kubernetes.io/hostname
containers:
- args:
- -port=8443
env:
- name: AWS_REGION
value: us-test-1
- name: CLUSTER_NAME
value: minimal.example.com
- env:
- name: KUBERNETES_MIN_VERSION
value: 1.19.0-0
- name: CLUSTER_ENDPOINT
value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME
value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
value: karpenter
- name: WEBHOOK_PORT
value: "8443"
- name: METRICS_PORT
value: "8080"
- name: HEALTH_PROBE_PORT
value: "8081"
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: MEMORY_LIMIT
valueFrom:
resourceFieldRef:
containerName: controller
divisor: "0"
resource: limits.memory
- name: AWS_REGION
value: us-test-1
- name: AWS_ROLE_ARN
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/webhook:v0.16.3
image: public.ecr.aws/karpenter/controller:v0.27.5
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
port: https-webhook
scheme: HTTPS
path: /healthz
port: http
initialDelaySeconds: 30
name: webhook
timeoutSeconds: 30
name: controller
ports:
- containerPort: 8080
name: http-metrics
protocol: TCP
- containerPort: 8081
name: http
protocol: TCP
- containerPort: 8443
name: https-webhook
protocol: TCP
readinessProbe:
httpGet:
port: https-webhook
scheme: HTTPS
path: /readyz
port: http
timeoutSeconds: 30
resources:
limits:
cpu: 100m
memory: 50Mi
memory: 1Gi
requests:
cpu: 100m
memory: 50Mi
startupProbe:
failureThreshold: 6
httpGet:
port: https-webhook
scheme: HTTPS
cpu: 500m
memory: 1Gi
volumeMounts:
- mountPath: /var/run/secrets/amazonaws.com/
name: token-amazonaws-com
@ -1280,7 +1254,7 @@ spec:
dnsPolicy: Default
priorityClassName: system-cluster-critical
securityContext:
fsGroup: 10001
fsGroup: 1000
serviceAccountName: karpenter
tolerations:
- key: node-role.kubernetes.io/master
@ -1290,12 +1264,16 @@ spec:
topologySpreadConstraints:
- labelSelector:
matchLabels:
app.kubernetes.io/instance: karpenter
app.kubernetes.io/name: karpenter
karpenter: webhook
maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
- labelSelector:
matchLabels:
app.kubernetes.io/instance: karpenter
app.kubernetes.io/name: karpenter
karpenter: webhook
maxSkew: 1
topologyKey: kubernetes.io/hostname
@ -1320,16 +1298,16 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: defaulting.webhook.provisioners.karpenter.sh
name: defaulting.webhook.karpenter.k8s.aws
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: karpenter-webhook
name: karpenter
namespace: kube-system
failurePolicy: Fail
name: defaulting.webhook.provisioners.karpenter.sh
name: defaulting.webhook.karpenter.k8s.aws
rules:
- apiGroups:
- karpenter.k8s.aws
@ -1364,16 +1342,16 @@ metadata:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: validation.webhook.provisioners.karpenter.sh
name: validation.webhook.karpenter.k8s.aws
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: karpenter-webhook
name: karpenter
namespace: kube-system
failurePolicy: Fail
name: validation.webhook.provisioners.karpenter.sh
name: validation.webhook.karpenter.k8s.aws
rules:
- apiGroups:
- karpenter.k8s.aws
@ -1382,7 +1360,6 @@ webhooks:
operations:
- CREATE
- UPDATE
- DELETE
resources:
- awsnodetemplates
- awsnodetemplates/status
@ -1394,7 +1371,39 @@ webhooks:
operations:
- CREATE
- UPDATE
- DELETE
resources:
- provisioners
- provisioners/status
sideEffects: None
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: validation.webhook.karpenter.sh
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: karpenter
namespace: kube-system
failurePolicy: Fail
name: validation.webhook.karpenter.sh
rules:
- apiGroups:
- karpenter.sh
apiVersions:
- v1alpha5
operations:
- CREATE
- UPDATE
resources:
- provisioners
- provisioners/status
@ -1416,7 +1425,7 @@ webhooks:
- v1
clientConfig:
service:
name: karpenter-webhook
name: karpenter
namespace: kube-system
failurePolicy: Fail
name: validation.webhook.config.karpenter.sh
@ -1436,6 +1445,7 @@ metadata:
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter
namespace: kube-system
spec:
maxUnavailable: 1
selector:
@ -1444,6 +1454,23 @@ spec:
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-nodes-default
spec:
launchTemplate: karpenter-nodes-default.minimal.example.com
subnetSelector:
kops.k8s.io/instance-group/karpenter-nodes-default: '*'
kubernetes.io/cluster/minimal.example.com: '*'
---
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
@ -1464,11 +1491,8 @@ spec:
systemReserved:
cpu: 500m
memory: 1G
provider:
launchTemplate: karpenter-nodes-default.minimal.example.com
subnetSelector:
kops.k8s.io/instance-group/karpenter-nodes-default: '*'
kubernetes.io/cluster/minimal.example.com: '*'
providerRef:
name: karpenter-nodes-default
requirements:
- key: karpenter.sh/capacity-type
operator: In
@ -1489,6 +1513,23 @@ spec:
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter-nodes-single-machinetype
spec:
launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com
subnetSelector:
kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*'
kubernetes.io/cluster/minimal.example.com: '*'
---
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
@ -1501,11 +1542,8 @@ metadata:
spec:
consolidation:
enabled: true
provider:
launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com
subnetSelector:
kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*'
kubernetes.io/cluster/minimal.example.com: '*'
providerRef:
name: karpenter-nodes-single-machinetype
requirements:
- key: karpenter.sh/capacity-type
operator: In

855
upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template
View File

File diff suppressed because it is too large Load Diff

3
upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go
View File

@ -1232,7 +1232,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
{
id := "k8s-1.19"
location := key + "/" + id + ".yaml"
addons.Add(&channelsapi.AddonSpec{
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Manifest: fi.PtrTo(location),
Selector: map[string]string{"k8s-addon": key},
@ -1241,6 +1241,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
if b.UseServiceAccountExternalPermissions() {
serviceAccountRoles = append(serviceAccountRoles, &karpenter.ServiceAccount{})
}
addon.BuildPrune = true
}
}