diff --git a/pkg/model/components/addonmanifests/karpenter/iam.go b/pkg/model/components/addonmanifests/karpenter/iam.go index 335c164027..e5646bf9aa 100644 --- a/pkg/model/components/addonmanifests/karpenter/iam.go +++ b/pkg/model/components/addonmanifests/karpenter/iam.go @@ -48,7 +48,9 @@ func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { func addKarpenterPermissions(p *iam.Policy) { p.AddUnconditionalActions( - // "ec2:CreateLaunchTemplate", + // Not included because we require Karpenter + // use existing kOps instance group launch templates + // "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:RunInstances", "ec2:CreateTags", diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index d2a6eb6b90..d100661add 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: d20d4eda23c530caceacaddc88f3ef89bb2228edbdf0e9a8854950282c64c648 + manifestHash: 972350ca292e37ca46ad54460607f1492a2311ff0b074e2c62258bacc2cffff1 name: karpenter.sh selector: k8s-addon: karpenter.sh diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index c79a4bcac2..182a2bb2c2 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -12,6 +12,8 @@ metadata: spec: group: karpenter.sh names: + categories: + - karpenter kind: Provisioner listKind: ProvisionerList plural: provisioners @@ -42,6 +44,17 @@ spec: Node properties are determined from a combination of provisioner and pod scheduling constraints. properties: + kubeletConfiguration: + description: KubeletConfiguration are options passed to the kubelet + when provisioning nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster + DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + type: object labels: additionalProperties: type: string @@ -242,6 +255,8 @@ metadata: apiVersion: v1 data: + loglevel.controller: debug + loglevel.webhook: debug zap-logger-config: | { "level": "debug", @@ -253,7 +268,7 @@ data: "thereafter": 100 }, "outputPaths": ["stdout"], - "errorOutputPaths": ["stderr"], + "errorOutputPaths": ["stderr"], "encoding": "console", "encoderConfig": { "timeKey": "time", @@ -293,6 +308,13 @@ rules: - karpenter.sh resources: - provisioners + verbs: + - get + - list + - watch +- apiGroups: + - karpenter.sh + resources: - provisioners/status verbs: - create @@ -302,14 +324,22 @@ rules: - list - watch - apiGroups: - - coordination.k8s.io + - "" resources: - - leases + - persistentvolumes + - persistentvolumeclaims verbs: - - create - get - - patch + - list + - watch - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list - watch - apiGroups: - "" @@ -326,13 +356,10 @@ rules: - "" resources: - configmaps - - persistentvolumes - - persistentvolumeclaims verbs: - get - list - watch - - update - apiGroups: - "" resources: @@ -473,7 +500,6 @@ rules: - apiGroups: - "" resources: - - configmaps - namespaces verbs: - get @@ -555,7 +581,7 @@ metadata: spec: ports: - port: 8080 - targetPort: metrics + targetPort: http-metrics selector: karpenter: controller @@ -574,7 +600,7 @@ metadata: spec: ports: - port: 443 - targetPort: webhook + targetPort: https-webhook selector: karpenter: webhook @@ -601,6 +627,7 @@ spec: metadata: creationTimestamp: null labels: + app.kubernetes.io/name: karpenter karpenter: controller kops.k8s.io/managed-by: kops spec: @@ -627,6 +654,8 @@ spec: value: https://api.internal.minimal.example.com - name: CONFIG_LOGGING_NAME value: karpenter-config-logging + - name: KARPENTER_SERVICE + value: karpenter-webhook - name: SYSTEM_NAMESPACE valueFrom: fieldRef: @@ -635,21 +664,24 @@ spec: value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com - name: AWS_WEB_IDENTITY_TOKEN_FILE value: /var/run/secrets/amazonaws.com/token - image: public.ecr.aws/karpenter/controller:v0.6.0 + image: public.ecr.aws/karpenter/controller:v0.10.0 + imagePullPolicy: IfNotPresent livenessProbe: httpGet: path: /healthz - port: 8081 - name: manager + port: http + name: controller ports: - containerPort: 8080 - name: metrics + name: http-metrics + protocol: TCP - containerPort: 8081 - name: health-probe + name: http + protocol: TCP readinessProbe: httpGet: path: /readyz - port: 8081 + port: http resources: limits: memory: 1Gi @@ -716,6 +748,7 @@ spec: metadata: creationTimestamp: null labels: + app.kubernetes.io/name: karpenter karpenter: webhook kops.k8s.io/managed-by: kops spec: @@ -749,10 +782,14 @@ spec: value: us-test-1 - name: CLUSTER_NAME value: minimal.example.com + - name: KUBERNETES_MIN_VERSION + value: 1.19.0-0 - name: CLUSTER_ENDPOINT value: https://api.internal.minimal.example.com - name: CONFIG_LOGGING_NAME value: karpenter-config-logging + - name: KARPENTER_SERVICE + value: karpenter-webhook - name: SYSTEM_NAMESPACE valueFrom: fieldRef: @@ -761,18 +798,20 @@ spec: value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com - name: AWS_WEB_IDENTITY_TOKEN_FILE value: /var/run/secrets/amazonaws.com/token - image: public.ecr.aws/karpenter/webhook:v0.6.0 + image: public.ecr.aws/karpenter/webhook:v0.10.0 + imagePullPolicy: IfNotPresent livenessProbe: httpGet: - port: 8443 + port: https-webhook scheme: HTTPS name: webhook ports: - containerPort: 8443 - name: webhook + name: https-webhook + protocol: TCP readinessProbe: httpGet: - port: 8443 + port: https-webhook scheme: HTTPS resources: limits: @@ -781,10 +820,16 @@ spec: requests: cpu: 100m memory: 50Mi + startupProbe: + failureThreshold: 6 + httpGet: + port: https-webhook + scheme: HTTPS volumeMounts: - mountPath: /var/run/secrets/amazonaws.com/ name: token-amazonaws-com readOnly: true + dnsPolicy: Default priorityClassName: system-cluster-critical securityContext: fsGroup: 10001 @@ -846,7 +891,8 @@ webhooks: - CREATE - UPDATE resources: - - provisioners provisioners/status + - provisioners + - provisioners/status sideEffects: None --- @@ -877,8 +923,10 @@ webhooks: operations: - CREATE - UPDATE + - DELETE resources: - - provisioners provisioners/status + - provisioners + - provisioners/status sideEffects: None --- @@ -908,6 +956,23 @@ webhooks: --- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: karpenter + +--- + apiVersion: karpenter.sh/v1alpha5 kind: Provisioner metadata: @@ -919,10 +984,7 @@ metadata: name: karpenter-nodes-default spec: provider: - instanceProfile: nodes.minimal.example.com launchTemplate: karpenter-nodes-default.minimal.example.com - securityGroupSelector: - Name: nodes.minimal.example.com subnetSelector: kubernetes.io/cluster/minimal.example.com: '*' kubernetes.io/role/internal-elb: "1" @@ -954,10 +1016,7 @@ metadata: name: karpenter-nodes-single-machinetype spec: provider: - instanceProfile: nodes.minimal.example.com launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com - securityGroupSelector: - Name: nodes.minimal.example.com subnetSelector: kubernetes.io/cluster/minimal.example.com: '*' kubernetes.io/role/internal-elb: "1" diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 227ffd4bac..dd60f509f5 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -10,6 +10,8 @@ metadata: spec: group: karpenter.sh names: + categories: + - karpenter kind: Provisioner listKind: ProvisionerList plural: provisioners @@ -40,6 +42,17 @@ spec: Node properties are determined from a combination of provisioner and pod scheduling constraints. properties: + kubeletConfiguration: + description: KubeletConfiguration are options passed to the kubelet + when provisioning nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster + DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + type: object labels: additionalProperties: type: string @@ -246,7 +259,7 @@ data: "thereafter": 100 }, "outputPaths": ["stdout"], - "errorOutputPaths": ["stderr"], + "errorOutputPaths": ["stderr"], "encoding": "console", "encoderConfig": { "timeKey": "time", @@ -260,8 +273,8 @@ data: } } # Log level overrides - # loglevel.controller: info # debug - # loglevel.webhook: info # debug + loglevel.controller: debug + loglevel.webhook: debug --- # Source: karpenter/templates/controller/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -270,17 +283,23 @@ metadata: name: karpenter-controller rules: - apiGroups: ["karpenter.sh"] - resources: ["provisioners", "provisioners/status"] + resources: ["provisioners"] + verbs: ["get", "list", "watch"] +- apiGroups: ["karpenter.sh"] + resources: ["provisioners/status"] verbs: ["create", "delete", "patch", "get", "list", "watch"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "get", "patch", "update", "watch"] +- apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes", "pods"] verbs: ["get", "list", "watch", "patch", "delete"] - apiGroups: [""] - resources: ["configmaps", "persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["create"] @@ -354,7 +373,7 @@ metadata: namespace: kube-system rules: - apiGroups: [""] - resources: ["configmaps", "namespaces"] + resources: ["namespaces"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] @@ -402,7 +421,7 @@ metadata: spec: ports: - port: 8080 - targetPort: metrics + targetPort: http-metrics selector: karpenter: controller --- @@ -415,7 +434,7 @@ metadata: spec: ports: - port: 443 - targetPort: webhook + targetPort: https-webhook selector: karpenter: webhook --- @@ -436,6 +455,7 @@ spec: metadata: labels: karpenter: controller + app.kubernetes.io/name: karpenter spec: affinity: nodeAffinity: @@ -451,27 +471,30 @@ spec: serviceAccountName: karpenter dnsPolicy: Default containers: - - name: manager - image: public.ecr.aws/karpenter/controller:v0.6.0 - resources: + - name: controller + image: public.ecr.aws/karpenter/controller:v0.10.0 + imagePullPolicy: IfNotPresent + resources: limits: memory: 1Gi requests: cpu: 500m memory: 1Gi ports: - - name: metrics + - name: http-metrics containerPort: 8080 - - name: health-probe + protocol: TCP + - name: http containerPort: 8081 + protocol: TCP livenessProbe: httpGet: path: /healthz - port: 8081 + port: http readinessProbe: httpGet: path: /readyz - port: 8081 + port: http env: {{ if not .Networking.AmazonVPC }} - name: AWS_ENI_LIMITED_POD_DENSITY @@ -489,6 +512,8 @@ spec: value: https://{{ .MasterInternalName }} - name: CONFIG_LOGGING_NAME value: "karpenter-config-logging" + - name: KARPENTER_SERVICE + value: karpenter-webhook - name: SYSTEM_NAMESPACE valueFrom: fieldRef: @@ -540,13 +565,16 @@ spec: template: metadata: labels: + app.kubernetes.io/name: karpenter karpenter: webhook spec: priorityClassName: system-cluster-critical serviceAccountName: karpenter + dnsPolicy: Default containers: - name: webhook - image: public.ecr.aws/karpenter/webhook:v0.6.0 + image: public.ecr.aws/karpenter/webhook:v0.10.0 + imagePullPolicy: IfNotPresent args: - -port=8443 resources: @@ -557,25 +585,35 @@ spec: cpu: 100m memory: 50Mi ports: - - name: webhook + - name: https-webhook containerPort: 8443 + protocol: TCP livenessProbe: httpGet: scheme: HTTPS - port: 8443 + port: https-webhook readinessProbe: httpGet: scheme: HTTPS - port: 8443 + port: https-webhook + startupProbe: + httpGet: + scheme: HTTPS + port: https-webhook + failureThreshold: 6 env: - name: AWS_REGION value: {{ Region }} - name: CLUSTER_NAME value: {{ ClusterName }} + - name: KUBERNETES_MIN_VERSION + value: "1.19.0-0" - name: CLUSTER_ENDPOINT value: https://{{ .MasterInternalName }} - name: CONFIG_LOGGING_NAME value: "karpenter-config-logging" + - name: KARPENTER_SERVICE + value: karpenter-webhook - name: SYSTEM_NAMESPACE valueFrom: fieldRef: @@ -609,7 +647,7 @@ spec: operator: Exists topologySpreadConstraints: - maxSkew: 1 - topologyKey: "topology.kubernetes.io/zone" + topologyKey: topology.kubernetes.io/zone whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: @@ -642,7 +680,7 @@ webhooks: - v1alpha5 resources: - provisioners - provisioners/status + - provisioners/status operations: - CREATE - UPDATE @@ -668,10 +706,11 @@ webhooks: - v1alpha5 resources: - provisioners - provisioners/status + - provisioners/status operations: - CREATE - UPDATE + - DELETE --- # Source: karpenter/templates/webhook/webhooks.yaml apiVersion: admissionregistration.k8s.io/v1 @@ -690,6 +729,17 @@ webhooks: objectSelector: matchLabels: app.kubernetes.io/part-of: karpenter +--- +# Source: karpenter/templates/poddisruptionbudget.yaml +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: karpenter +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: karpenter {{ range $name, $spec := GetNodeInstanceGroups }} {{ if eq $spec.Manager "Karpenter" }} @@ -730,10 +780,7 @@ spec: {{ end }} {{ end }} provider: - instanceProfile: nodes.{{ ClusterName }} launchTemplate: {{ $name }}.{{ ClusterName }} - securityGroupSelector: - Name: nodes.{{ ClusterName }} subnetSelector: kubernetes.io/role/internal-elb: "1" kubernetes.io/cluster/{{ ClusterName }}: "*"