kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update Karpenter to v0.10.0 

./hack/update-expected.sh

Use default dns policy for webhook

Fix webhook svc target port

Fix provisioner to only contain launchTemplate
This commit is contained in:
Peter Rifel 2022-05-09 19:11:26 -05:00 committed by Ole Markus With
parent b9944bea51
commit ef3a96558f
4 changed files with 168 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/model/components/addonmanifests/karpenter/iam.go
View File

@ -48,7 +48,9 @@ func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) {
func addKarpenterPermissions(p *iam.Policy) {
p.AddUnconditionalActions(
// "ec2:CreateLaunchTemplate",
// Not included because we require Karpenter
// use existing kOps instance group launch templates
// "ec2:CreateLaunchTemplate",
"ec2:CreateFleet",
"ec2:RunInstances",
"ec2:CreateTags",

2
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
View File

@ -54,7 +54,7 @@ spec:
version: 9.99.0
- id: k8s-1.19
manifest: karpenter.sh/k8s-1.19.yaml
manifestHash: d20d4eda23c530caceacaddc88f3ef89bb2228edbdf0e9a8854950282c64c648
manifestHash: 972350ca292e37ca46ad54460607f1492a2311ff0b074e2c62258bacc2cffff1
name: karpenter.sh
selector:
k8s-addon: karpenter.sh

117
tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content
View File

@ -12,6 +12,8 @@ metadata:
spec:
group: karpenter.sh
names:
categories:
- karpenter
kind: Provisioner
listKind: ProvisionerList
plural: provisioners
@ -42,6 +44,17 @@ spec:
Node properties are determined from a combination of provisioner and
pod scheduling constraints.
properties:
kubeletConfiguration:
description: KubeletConfiguration are options passed to the kubelet
when provisioning nodes
properties:
clusterDNS:
description: clusterDNS is a list of IP addresses for the cluster
DNS server. Note that not all providers may use all addresses.
items:
type: string
type: array
type: object
labels:
additionalProperties:
type: string
@ -242,6 +255,8 @@ metadata:
apiVersion: v1
data:
loglevel.controller: debug
loglevel.webhook: debug
zap-logger-config: |
{
"level": "debug",
@ -253,7 +268,7 @@ data:
"thereafter": 100
},
"outputPaths": ["stdout"],
"errorOutputPaths": ["stderr"],
"errorOutputPaths": ["stderr"],
"encoding": "console",
"encoderConfig": {
"timeKey": "time",
@ -293,6 +308,13 @@ rules:
- karpenter.sh
resources:
- provisioners
verbs:
- get
- list
- watch
- apiGroups:
- karpenter.sh
resources:
- provisioners/status
verbs:
- create
@ -302,14 +324,22 @@ rules:
- list
- watch
- apiGroups:
- coordination.k8s.io
- ""
resources:
- leases
- persistentvolumes
- persistentvolumeclaims
verbs:
- create
- get
- patch
- list
- watch
- update
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
@ -326,13 +356,10 @@ rules:
- ""
resources:
- configmaps
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
@ -473,7 +500,6 @@ rules:
- apiGroups:
- ""
resources:
- configmaps
- namespaces
verbs:
- get
@ -555,7 +581,7 @@ metadata:
spec:
ports:
- port: 8080
targetPort: metrics
targetPort: http-metrics
selector:
karpenter: controller
@ -574,7 +600,7 @@ metadata:
spec:
ports:
- port: 443
targetPort: webhook
targetPort: https-webhook
selector:
karpenter: webhook
@ -601,6 +627,7 @@ spec:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: karpenter
karpenter: controller
kops.k8s.io/managed-by: kops
spec:
@ -627,6 +654,8 @@ spec:
value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME
value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
@ -635,21 +664,24 @@ spec:
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/controller:v0.6.0
image: public.ecr.aws/karpenter/controller:v0.10.0
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /healthz
port: 8081
name: manager
port: http
name: controller
ports:
- containerPort: 8080
name: metrics
name: http-metrics
protocol: TCP
- containerPort: 8081
name: health-probe
name: http
protocol: TCP
readinessProbe:
httpGet:
path: /readyz
port: 8081
port: http
resources:
limits:
memory: 1Gi
@ -716,6 +748,7 @@ spec:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: karpenter
karpenter: webhook
kops.k8s.io/managed-by: kops
spec:
@ -749,10 +782,14 @@ spec:
value: us-test-1
- name: CLUSTER_NAME
value: minimal.example.com
- name: KUBERNETES_MIN_VERSION
value: 1.19.0-0
- name: CLUSTER_ENDPOINT
value: https://api.internal.minimal.example.com
- name: CONFIG_LOGGING_NAME
value: karpenter-config-logging
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
@ -761,18 +798,20 @@ spec:
value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com
- name: AWS_WEB_IDENTITY_TOKEN_FILE
value: /var/run/secrets/amazonaws.com/token
image: public.ecr.aws/karpenter/webhook:v0.6.0
image: public.ecr.aws/karpenter/webhook:v0.10.0
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
port: 8443
port: https-webhook
scheme: HTTPS
name: webhook
ports:
- containerPort: 8443
name: webhook
name: https-webhook
protocol: TCP
readinessProbe:
httpGet:
port: 8443
port: https-webhook
scheme: HTTPS
resources:
limits:
@ -781,10 +820,16 @@ spec:
requests:
cpu: 100m
memory: 50Mi
startupProbe:
failureThreshold: 6
httpGet:
port: https-webhook
scheme: HTTPS
volumeMounts:
- mountPath: /var/run/secrets/amazonaws.com/
name: token-amazonaws-com
readOnly: true
dnsPolicy: Default
priorityClassName: system-cluster-critical
securityContext:
fsGroup: 10001
@ -846,7 +891,8 @@ webhooks:
- CREATE
- UPDATE
resources:
- provisioners provisioners/status
- provisioners
- provisioners/status
sideEffects: None
---
@ -877,8 +923,10 @@ webhooks:
operations:
- CREATE
- UPDATE
- DELETE
resources:
- provisioners provisioners/status
- provisioners
- provisioners/status
sideEffects: None
---
@ -908,6 +956,23 @@ webhooks:
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: karpenter.sh
app.kubernetes.io/managed-by: kops
k8s-addon: karpenter.sh
name: karpenter
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: karpenter
---
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
@ -919,10 +984,7 @@ metadata:
name: karpenter-nodes-default
spec:
provider:
instanceProfile: nodes.minimal.example.com
launchTemplate: karpenter-nodes-default.minimal.example.com
securityGroupSelector:
Name: nodes.minimal.example.com
subnetSelector:
kubernetes.io/cluster/minimal.example.com: '*'
kubernetes.io/role/internal-elb: "1"
@ -954,10 +1016,7 @@ metadata:
name: karpenter-nodes-single-machinetype
spec:
provider:
instanceProfile: nodes.minimal.example.com
launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com
securityGroupSelector:
Name: nodes.minimal.example.com
subnetSelector:
kubernetes.io/cluster/minimal.example.com: '*'
kubernetes.io/role/internal-elb: "1"

105
upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template
View File

@ -10,6 +10,8 @@ metadata:
spec:
group: karpenter.sh
names:
categories:
- karpenter
kind: Provisioner
listKind: ProvisionerList
plural: provisioners
@ -40,6 +42,17 @@ spec:
Node properties are determined from a combination of provisioner and
pod scheduling constraints.
properties:
kubeletConfiguration:
description: KubeletConfiguration are options passed to the kubelet
when provisioning nodes
properties:
clusterDNS:
description: clusterDNS is a list of IP addresses for the cluster
DNS server. Note that not all providers may use all addresses.
items:
type: string
type: array
type: object
labels:
additionalProperties:
type: string
@ -246,7 +259,7 @@ data:
"thereafter": 100
},
"outputPaths": ["stdout"],
"errorOutputPaths": ["stderr"],
"errorOutputPaths": ["stderr"],
"encoding": "console",
"encoderConfig": {
"timeKey": "time",
@ -260,8 +273,8 @@ data:
}
}
# Log level overrides
# loglevel.controller: info # debug
# loglevel.webhook: info # debug
loglevel.controller: debug
loglevel.webhook: debug
---
# Source: karpenter/templates/controller/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -270,17 +283,23 @@ metadata:
name: karpenter-controller
rules:
- apiGroups: ["karpenter.sh"]
resources: ["provisioners", "provisioners/status"]
resources: ["provisioners"]
verbs: ["get", "list", "watch"]
- apiGroups: ["karpenter.sh"]
resources: ["provisioners/status"]
verbs: ["create", "delete", "patch", "get", "list", "watch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["create", "get", "patch", "update", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes", "persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["nodes", "pods"]
verbs: ["get", "list", "watch", "patch", "delete"]
- apiGroups: [""]
resources: ["configmaps", "persistentvolumes", "persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
resources: ["configmaps"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["create"]
@ -354,7 +373,7 @@ metadata:
namespace: kube-system
rules:
- apiGroups: [""]
resources: ["configmaps", "namespaces"]
resources: ["namespaces"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
@ -402,7 +421,7 @@ metadata:
spec:
ports:
- port: 8080
targetPort: metrics
targetPort: http-metrics
selector:
karpenter: controller
---
@ -415,7 +434,7 @@ metadata:
spec:
ports:
- port: 443
targetPort: webhook
targetPort: https-webhook
selector:
karpenter: webhook
---
@ -436,6 +455,7 @@ spec:
metadata:
labels:
karpenter: controller
app.kubernetes.io/name: karpenter
spec:
affinity:
nodeAffinity:
@ -451,27 +471,30 @@ spec:
serviceAccountName: karpenter
dnsPolicy: Default
containers:
- name: manager
image: public.ecr.aws/karpenter/controller:v0.6.0
resources:
- name: controller
image: public.ecr.aws/karpenter/controller:v0.10.0
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
ports:
- name: metrics
- name: http-metrics
containerPort: 8080
- name: health-probe
protocol: TCP
- name: http
containerPort: 8081
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: 8081
port: http
readinessProbe:
httpGet:
path: /readyz
port: 8081
port: http
env:
{{ if not .Networking.AmazonVPC }}
- name: AWS_ENI_LIMITED_POD_DENSITY
@ -489,6 +512,8 @@ spec:
value: https://{{ .MasterInternalName }}
- name: CONFIG_LOGGING_NAME
value: "karpenter-config-logging"
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
@ -540,13 +565,16 @@ spec:
template:
metadata:
labels:
app.kubernetes.io/name: karpenter
karpenter: webhook
spec:
priorityClassName: system-cluster-critical
serviceAccountName: karpenter
dnsPolicy: Default
containers:
- name: webhook
image: public.ecr.aws/karpenter/webhook:v0.6.0
image: public.ecr.aws/karpenter/webhook:v0.10.0
imagePullPolicy: IfNotPresent
args:
- -port=8443
resources:
@ -557,25 +585,35 @@ spec:
cpu: 100m
memory: 50Mi
ports:
- name: webhook
- name: https-webhook
containerPort: 8443
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTPS
port: 8443
port: https-webhook
readinessProbe:
httpGet:
scheme: HTTPS
port: 8443
port: https-webhook
startupProbe:
httpGet:
scheme: HTTPS
port: https-webhook
failureThreshold: 6
env:
- name: AWS_REGION
value: {{ Region }}
- name: CLUSTER_NAME
value: {{ ClusterName }}
- name: KUBERNETES_MIN_VERSION
value: "1.19.0-0"
- name: CLUSTER_ENDPOINT
value: https://{{ .MasterInternalName }}
- name: CONFIG_LOGGING_NAME
value: "karpenter-config-logging"
- name: KARPENTER_SERVICE
value: karpenter-webhook
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
@ -609,7 +647,7 @@ spec:
operator: Exists
topologySpreadConstraints:
- maxSkew: 1
topologyKey: "topology.kubernetes.io/zone"
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
@ -642,7 +680,7 @@ webhooks:
- v1alpha5
resources:
- provisioners
provisioners/status
- provisioners/status
operations:
- CREATE
- UPDATE
@ -668,10 +706,11 @@ webhooks:
- v1alpha5
resources:
- provisioners
provisioners/status
- provisioners/status
operations:
- CREATE
- UPDATE
- DELETE
---
# Source: karpenter/templates/webhook/webhooks.yaml
apiVersion: admissionregistration.k8s.io/v1
@ -690,6 +729,17 @@ webhooks:
objectSelector:
matchLabels:
app.kubernetes.io/part-of: karpenter
---
# Source: karpenter/templates/poddisruptionbudget.yaml
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: karpenter
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: karpenter
{{ range $name, $spec := GetNodeInstanceGroups }}
{{ if eq $spec.Manager "Karpenter" }}
@ -730,10 +780,7 @@ spec:
{{ end }}
{{ end }}
provider:
instanceProfile: nodes.{{ ClusterName }}
launchTemplate: {{ $name }}.{{ ClusterName }}
securityGroupSelector:
Name: nodes.{{ ClusterName }}
subnetSelector:
kubernetes.io/role/internal-elb: "1"
kubernetes.io/cluster/{{ ClusterName }}: "*"