kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update Karpenter to v0.10.0 

./hack/update-expected.sh

Use default dns policy for webhook

Fix webhook svc target port

Fix provisioner to only contain launchTemplate
This commit is contained in:
Peter Rifel 2022-05-09 19:11:26 -05:00 committed by Ole Markus With
parent b9944bea51
commit ef3a96558f
4 changed files with 168 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/model/components/addonmanifests/karpenter/iam.go
View File

@ -48,7 +48,9 @@ func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) {
func addKarpenterPermissions(p *iam.Policy) { func addKarpenterPermissions(p *iam.Policy) {
p.AddUnconditionalActions( p.AddUnconditionalActions(
// "ec2:CreateLaunchTemplate", // Not included because we require Karpenter
// use existing kOps instance group launch templates
// "ec2:CreateLaunchTemplate",
"ec2:CreateFleet", "ec2:CreateFleet",
"ec2:RunInstances", "ec2:RunInstances",
"ec2:CreateTags", "ec2:CreateTags",

2
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
View File

@ -54,7 +54,7 @@ spec:
version: 9.99.0 version: 9.99.0
- id: k8s-1.19 - id: k8s-1.19
manifest: karpenter.sh/k8s-1.19.yaml manifest: karpenter.sh/k8s-1.19.yaml
manifestHash: d20d4eda23c530caceacaddc88f3ef89bb2228edbdf0e9a8854950282c64c648 manifestHash: 972350ca292e37ca46ad54460607f1492a2311ff0b074e2c62258bacc2cffff1
name: karpenter.sh name: karpenter.sh
selector: selector:
k8s-addon: karpenter.sh k8s-addon: karpenter.sh

117
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content
View File

@ -12,6 +12,8 @@ metadata:
spec: spec:
group: karpenter.sh group: karpenter.sh
names: names:
categories:
- karpenter
kind: Provisioner kind: Provisioner
listKind: ProvisionerList listKind: ProvisionerList
plural: provisioners plural: provisioners
@ -42,6 +44,17 @@ spec:
Node properties are determined from a combination of provisioner and Node properties are determined from a combination of provisioner and
pod scheduling constraints. pod scheduling constraints.
properties: properties:
kubeletConfiguration:
description: KubeletConfiguration are options passed to the kubelet
when provisioning nodes
properties:
clusterDNS:
description: clusterDNS is a list of IP addresses for the cluster
DNS server. Note that not all providers may use all addresses.
items:
type: string
type: array
type: object
labels: labels:
additionalProperties: additionalProperties:
type: string type: string
@ -242,6 +255,8 @@ metadata:
apiVersion: v1 apiVersion: v1
data: data:
loglevel.controller: debug
loglevel.webhook: debug
zap-logger-config: | zap-logger-config: |
{ {
"level": "debug", "level": "debug",
@ -253,7 +268,7 @@ data:
"thereafter": 100 "thereafter": 100
}, },
"outputPaths": ["stdout"], "outputPaths": ["stdout"],
"errorOutputPaths": ["stderr"], "errorOutputPaths": ["stderr"],
"encoding": "console", "encoding": "console",
"encoderConfig": { "encoderConfig": {
"timeKey": "time", "timeKey": "time",
@ -293,6 +308,13 @@ rules:
- karpenter.sh - karpenter.sh
resources: resources:
- provisioners - provisioners
verbs:
- get
- list
- watch
- apiGroups:
- karpenter.sh
resources:
- provisioners/status - provisioners/status
verbs: verbs:
- create - create
@ -302,14 +324,22 @@ rules:
- list - list
- watch - watch
- apiGroups: - apiGroups:
- coordination.k8s.io - ""
resources: resources:
- leases - persistentvolumes
- persistentvolumeclaims
verbs: verbs:
- create
- get - get
- patch - list
- watch
- update - update
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch - watch
- apiGroups: - apiGroups:
- "" - ""
@ -326,13 +356,10 @@ rules:
- "" - ""
resources: resources:
- configmaps - configmaps
- persistentvolumes
- persistentvolumeclaims
verbs: verbs:
- get - get
- list - list
- watch - watch
- update
- apiGroups: - apiGroups:
- "" - ""
resources: resources:
@ -473,7 +500,6 @@ rules:
- apiGroups: - apiGroups:
- "" - ""
resources: resources:
- configmaps
- namespaces - namespaces
verbs: verbs:
- get - get
@ -555,7 +581,7 @@ metadata:
spec: spec:
ports: ports:
- port: 8080 - port: 8080
targetPort: metrics targetPort: http-metrics
selector: selector:
karpenter: controller karpenter: controller
@ -574,7 +600,7 @@ metadata:
spec: spec:
ports: ports:
- port: 443 - port: 443
targetPort: webhook targetPort: https-webhook
selector: selector:
karpenter: webhook karpenter: webhook
@ -601,6 +627,7 @@ spec:
metadata: metadata:
creationTimestamp: null creationTimestamp: null
labels: labels:
app.kubernetes.io/name: karpenter
karpenter: controller karpenter: controller
kops.k8s.io/managed-by: kops kops.k8s.io/managed-by: kops
spec: spec:
@ -627,6 +654,8 @@ spec:
value: https://api.internal.minimal.example.com value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME - name: CONFIG_LOGGING_NAME
value: karpenter-config-logging value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE - name: SYSTEM_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -635,21 +664,24 @@ spec:
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE - name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/controller:v0.6.0 image: public.ecr.aws/karpenter/controller:v0.10.0
imagePullPolicy: IfNotPresent
livenessProbe: livenessProbe:
httpGet: httpGet:
path: /healthz path: /healthz
port: 8081 port: http
name: manager name: controller
ports: ports:
- containerPort: 8080 - containerPort: 8080
name: metrics name: http-metrics
protocol: TCP
- containerPort: 8081 - containerPort: 8081
name: health-probe name: http
protocol: TCP
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /readyz path: /readyz
port: 8081 port: http
resources: resources:
limits: limits:
memory: 1Gi memory: 1Gi
@ -716,6 +748,7 @@ spec:
metadata: metadata:
creationTimestamp: null creationTimestamp: null
labels: labels:
app.kubernetes.io/name: karpenter
karpenter: webhook karpenter: webhook
kops.k8s.io/managed-by: kops kops.k8s.io/managed-by: kops
spec: spec:
@ -749,10 +782,14 @@ spec:
value: us-test-1 value: us-test-1
- name: CLUSTER_NAME - name: CLUSTER_NAME
value: minimal.example.com value: minimal.example.com
- name: KUBERNETES_MIN_VERSION
value: 1.19.0-0
- name: CLUSTER_ENDPOINT - name: CLUSTER_ENDPOINT
value: https://api.internal.minimal.example.com value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME - name: CONFIG_LOGGING_NAME
value: karpenter-config-logging value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE - name: SYSTEM_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -761,18 +798,20 @@ spec:
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE - name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/webhook:v0.6.0 image: public.ecr.aws/karpenter/webhook:v0.10.0
imagePullPolicy: IfNotPresent
livenessProbe: livenessProbe:
httpGet: httpGet:
port: 8443 port: https-webhook
scheme: HTTPS scheme: HTTPS
name: webhook name: webhook
ports: ports:
- containerPort: 8443 - containerPort: 8443
name: webhook name: https-webhook
protocol: TCP
readinessProbe: readinessProbe:
httpGet: httpGet:
port: 8443 port: https-webhook
scheme: HTTPS scheme: HTTPS
resources: resources:
limits: limits:
@ -781,10 +820,16 @@ spec:
requests: requests:
cpu: 100m cpu: 100m
memory: 50Mi memory: 50Mi
startupProbe:
failureThreshold: 6
httpGet:
port: https-webhook
scheme: HTTPS
volumeMounts: volumeMounts:
- mountPath: /var/run/secrets/amazonaws.com/ - mountPath: /var/run/secrets/amazonaws.com/
name: token-amazonaws-com name: token-amazonaws-com
readOnly: true readOnly: true
dnsPolicy: Default
priorityClassName: system-cluster-critical priorityClassName: system-cluster-critical
securityContext: securityContext:
fsGroup: 10001 fsGroup: 10001
@ -846,7 +891,8 @@ webhooks:
- CREATE - CREATE
- UPDATE - UPDATE
resources: resources:
- provisioners provisioners/status - provisioners
- provisioners/status
sideEffects: None sideEffects: None
--- ---
@ -877,8 +923,10 @@ webhooks:
operations: operations:
- CREATE - CREATE
- UPDATE - UPDATE
- DELETE
resources: resources:
- provisioners provisioners/status - provisioners
- provisioners/status
sideEffects: None sideEffects: None
--- ---
@ -908,6 +956,23 @@ webhooks:
--- ---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: karpenter
---
apiVersion: karpenter.sh/v1alpha5 apiVersion: karpenter.sh/v1alpha5
kind: Provisioner kind: Provisioner
metadata: metadata:
@ -919,10 +984,7 @@ metadata:
name: karpenter-nodes-default name: karpenter-nodes-default
spec: spec:
provider: provider:
instanceProfile: nodes.minimal.example.com
launchTemplate: karpenter-nodes-default.minimal.example.com launchTemplate: karpenter-nodes-default.minimal.example.com
securityGroupSelector:
Name: nodes.minimal.example.com
subnetSelector: subnetSelector:
kubernetes.io/cluster/minimal.example.com: '*' kubernetes.io/cluster/minimal.example.com: '*'
kubernetes.io/role/internal-elb: "1" kubernetes.io/role/internal-elb: "1"
@ -954,10 +1016,7 @@ metadata:
name: karpenter-nodes-single-machinetype name: karpenter-nodes-single-machinetype
spec: spec:
provider: provider:
instanceProfile: nodes.minimal.example.com
launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com
securityGroupSelector:
Name: nodes.minimal.example.com
subnetSelector: subnetSelector:
kubernetes.io/cluster/minimal.example.com: '*' kubernetes.io/cluster/minimal.example.com: '*'
kubernetes.io/role/internal-elb: "1" kubernetes.io/role/internal-elb: "1"

105
upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template
View File

@ -10,6 +10,8 @@ metadata:
spec: spec:
group: karpenter.sh group: karpenter.sh
names: names:
categories:
- karpenter
kind: Provisioner kind: Provisioner
listKind: ProvisionerList listKind: ProvisionerList
plural: provisioners plural: provisioners
@ -40,6 +42,17 @@ spec:
Node properties are determined from a combination of provisioner and Node properties are determined from a combination of provisioner and
pod scheduling constraints. pod scheduling constraints.
properties: properties:
kubeletConfiguration:
description: KubeletConfiguration are options passed to the kubelet
when provisioning nodes
properties:
clusterDNS:
description: clusterDNS is a list of IP addresses for the cluster
DNS server. Note that not all providers may use all addresses.
items:
type: string
type: array
type: object
labels: labels:
additionalProperties: additionalProperties:
type: string type: string
@ -246,7 +259,7 @@ data:
"thereafter": 100 "thereafter": 100
}, },
"outputPaths": ["stdout"], "outputPaths": ["stdout"],
"errorOutputPaths": ["stderr"], "errorOutputPaths": ["stderr"],
"encoding": "console", "encoding": "console",
"encoderConfig": { "encoderConfig": {
"timeKey": "time", "timeKey": "time",
@ -260,8 +273,8 @@ data:
} }
} }
# Log level overrides # Log level overrides
# loglevel.controller: info # debug loglevel.controller: debug
# loglevel.webhook: info # debug loglevel.webhook: debug
--- ---
# Source: karpenter/templates/controller/rbac.yaml # Source: karpenter/templates/controller/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
@ -270,17 +283,23 @@ metadata:
name: karpenter-controller name: karpenter-controller
rules: rules:
- apiGroups: ["karpenter.sh"] - apiGroups: ["karpenter.sh"]
resources: ["provisioners", "provisioners/status"] resources: ["provisioners"]
verbs: ["get", "list", "watch"]
- apiGroups: ["karpenter.sh"]
resources: ["provisioners/status"]
verbs: ["create", "delete", "patch", "get", "list", "watch"] verbs: ["create", "delete", "patch", "get", "list", "watch"]
- apiGroups: ["coordination.k8s.io"] - apiGroups: [""]
resources: ["leases"] resources: ["persistentvolumes", "persistentvolumeclaims"]
verbs: ["create", "get", "patch", "update", "watch"] verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""] - apiGroups: [""]
resources: ["nodes", "pods"] resources: ["nodes", "pods"]
verbs: ["get", "list", "watch", "patch", "delete"] verbs: ["get", "list", "watch", "patch", "delete"]
- apiGroups: [""] - apiGroups: [""]
resources: ["configmaps", "persistentvolumes", "persistentvolumeclaims"] resources: ["configmaps"]
verbs: ["get", "list", "watch", "update"] verbs: ["get", "list", "watch"]
- apiGroups: [""] - apiGroups: [""]
resources: ["nodes"] resources: ["nodes"]
verbs: ["create"] verbs: ["create"]
@ -354,7 +373,7 @@ metadata:
namespace: kube-system namespace: kube-system
rules: rules:
- apiGroups: [""] - apiGroups: [""]
resources: ["configmaps", "namespaces"] resources: ["namespaces"]
verbs: ["get", "list", "watch"] verbs: ["get", "list", "watch"]
- apiGroups: [""] - apiGroups: [""]
resources: ["secrets"] resources: ["secrets"]
@ -402,7 +421,7 @@ metadata:
spec: spec:
ports: ports:
- port: 8080 - port: 8080
targetPort: metrics targetPort: http-metrics
selector: selector:
karpenter: controller karpenter: controller
--- ---
@ -415,7 +434,7 @@ metadata:
spec: spec:
ports: ports:
- port: 443 - port: 443
targetPort: webhook targetPort: https-webhook
selector: selector:
karpenter: webhook karpenter: webhook
--- ---
@ -436,6 +455,7 @@ spec:
metadata: metadata:
labels: labels:
karpenter: controller karpenter: controller
app.kubernetes.io/name: karpenter
spec: spec:
affinity: affinity:
nodeAffinity: nodeAffinity:
@ -451,27 +471,30 @@ spec:
serviceAccountName: karpenter serviceAccountName: karpenter
dnsPolicy: Default dnsPolicy: Default
containers: containers:
- name: manager - name: controller
image: public.ecr.aws/karpenter/controller:v0.6.0 image: public.ecr.aws/karpenter/controller:v0.10.0
resources: imagePullPolicy: IfNotPresent
resources:
limits: limits:
memory: 1Gi memory: 1Gi
requests: requests:
cpu: 500m cpu: 500m
memory: 1Gi memory: 1Gi
ports: ports:
- name: metrics - name: http-metrics
containerPort: 8080 containerPort: 8080
- name: health-probe protocol: TCP
- name: http
containerPort: 8081 containerPort: 8081
protocol: TCP
livenessProbe: livenessProbe:
httpGet: httpGet:
path: /healthz path: /healthz
port: 8081 port: http
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /readyz path: /readyz
port: 8081 port: http
env: env:
{{ if not .Networking.AmazonVPC }} {{ if not .Networking.AmazonVPC }}
- name: AWS_ENI_LIMITED_POD_DENSITY - name: AWS_ENI_LIMITED_POD_DENSITY
@ -489,6 +512,8 @@ spec:
value: https://{{ .MasterInternalName }} value: https://{{ .MasterInternalName }}
- name: CONFIG_LOGGING_NAME - name: CONFIG_LOGGING_NAME
value: "karpenter-config-logging" value: "karpenter-config-logging"
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE - name: SYSTEM_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -540,13 +565,16 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app.kubernetes.io/name: karpenter
karpenter: webhook karpenter: webhook
spec: spec:
priorityClassName: system-cluster-critical priorityClassName: system-cluster-critical
serviceAccountName: karpenter serviceAccountName: karpenter
dnsPolicy: Default
containers: containers:
- name: webhook - name: webhook
image: public.ecr.aws/karpenter/webhook:v0.6.0 image: public.ecr.aws/karpenter/webhook:v0.10.0
imagePullPolicy: IfNotPresent
args: args:
- -port=8443 - -port=8443
resources: resources:
@ -557,25 +585,35 @@ spec:
cpu: 100m cpu: 100m
memory: 50Mi memory: 50Mi
ports: ports:
- name: webhook - name: https-webhook
containerPort: 8443 containerPort: 8443
protocol: TCP
livenessProbe: livenessProbe:
httpGet: httpGet:
scheme: HTTPS scheme: HTTPS
port: 8443 port: https-webhook
readinessProbe: readinessProbe:
httpGet: httpGet:
scheme: HTTPS scheme: HTTPS
port: 8443 port: https-webhook
startupProbe:
httpGet:
scheme: HTTPS
port: https-webhook
failureThreshold: 6
env: env:
- name: AWS_REGION - name: AWS_REGION
value: {{ Region }} value: {{ Region }}
- name: CLUSTER_NAME - name: CLUSTER_NAME
value: {{ ClusterName }} value: {{ ClusterName }}
- name: KUBERNETES_MIN_VERSION
value: "1.19.0-0"
- name: CLUSTER_ENDPOINT - name: CLUSTER_ENDPOINT
value: https://{{ .MasterInternalName }} value: https://{{ .MasterInternalName }}
- name: CONFIG_LOGGING_NAME - name: CONFIG_LOGGING_NAME
value: "karpenter-config-logging" value: "karpenter-config-logging"
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE - name: SYSTEM_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -609,7 +647,7 @@ spec:
operator: Exists operator: Exists
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: "topology.kubernetes.io/zone" topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway whenUnsatisfiable: ScheduleAnyway
labelSelector: labelSelector:
matchLabels: matchLabels:
@ -642,7 +680,7 @@ webhooks:
- v1alpha5 - v1alpha5
resources: resources:
- provisioners - provisioners
provisioners/status - provisioners/status
operations: operations:
- CREATE - CREATE
- UPDATE - UPDATE
@ -668,10 +706,11 @@ webhooks:
- v1alpha5 - v1alpha5
resources: resources:
- provisioners - provisioners
provisioners/status - provisioners/status
operations: operations:
- CREATE - CREATE
- UPDATE - UPDATE
- DELETE
--- ---
# Source: karpenter/templates/webhook/webhooks.yaml # Source: karpenter/templates/webhook/webhooks.yaml
apiVersion: admissionregistration.k8s.io/v1 apiVersion: admissionregistration.k8s.io/v1
@ -690,6 +729,17 @@ webhooks:
objectSelector: objectSelector:
matchLabels: matchLabels:
app.kubernetes.io/part-of: karpenter app.kubernetes.io/part-of: karpenter
---
# Source: karpenter/templates/poddisruptionbudget.yaml
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: karpenter
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: karpenter
{{ range $name, $spec := GetNodeInstanceGroups }} {{ range $name, $spec := GetNodeInstanceGroups }}
{{ if eq $spec.Manager "Karpenter" }} {{ if eq $spec.Manager "Karpenter" }}
@ -730,10 +780,7 @@ spec:
{{ end }} {{ end }}
{{ end }} {{ end }}
provider: provider:
instanceProfile: nodes.{{ ClusterName }}
launchTemplate: {{ $name }}.{{ ClusterName }} launchTemplate: {{ $name }}.{{ ClusterName }}
securityGroupSelector:
Name: nodes.{{ ClusterName }}
subnetSelector: subnetSelector:
kubernetes.io/role/internal-elb: "1" kubernetes.io/role/internal-elb: "1"
kubernetes.io/cluster/{{ ClusterName }}: "*" kubernetes.io/cluster/{{ ClusterName }}: "*"