From 325f5e8987e629c88c509529c787711944952e47 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 25 Aug 2022 19:35:50 +0300 Subject: [PATCH 1/3] Pass CSI driver manifest through kustomize --- .../k8s-1.22.yaml.template | 650 ++++++++++-------- 1 file changed, 367 insertions(+), 283 deletions(-) diff --git a/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template b/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template index ebed28fcad..e8ff927be9 100644 --- a/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template +++ b/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template @@ -6,28 +6,18 @@ metadata: name: hcloud-csi namespace: kube-system stringData: - token: {{ HCLOUD_TOKEN }} + token: '{{ HCLOUD_TOKEN }}' --- +allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.hetzner.cloud -spec: - attachRequired: true - podInfoOnMount: true - volumeLifecycleModes: - - Persistent ---- kind: StorageClass -apiVersion: storage.k8s.io/v1 metadata: - namespace: kube-system - name: hcloud-volumes annotations: storageclass.kubernetes.io/is-default-class: "true" + name: hcloud-volumes + namespace: kube-system provisioner: csi.hetzner.cloud volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true --- apiVersion: v1 kind: ServiceAccount @@ -35,184 +25,299 @@ metadata: name: hcloud-csi namespace: kube-system --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: hcloud-csi rules: - # attacher - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - # provisioner - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims", "persistentvolumeclaims/status"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - # resizer - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - # node - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: hcloud-csi -subjects: - - kind: ServiceAccount - name: hcloud-csi - namespace: kube-system roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: hcloud-csi - apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: hcloud-csi + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: hcloud-csi + name: hcloud-csi-controller-metrics + namespace: kube-system +spec: + ports: + - name: metrics + port: 9189 + targetPort: metrics + selector: + app: hcloud-csi-controller +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: hcloud-csi + name: hcloud-csi-node-metrics + namespace: kube-system +spec: + ports: + - name: metrics + port: 9189 + targetPort: metrics + selector: + app: hcloud-csi --- -kind: StatefulSet apiVersion: apps/v1 +kind: StatefulSet metadata: name: hcloud-csi-controller namespace: kube-system spec: + replicas: 1 selector: matchLabels: app: hcloud-csi-controller serviceName: hcloud-csi-controller - replicas: 1 template: metadata: labels: app: hcloud-csi-controller spec: - serviceAccount: hcloud-csi containers: - - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 - volumeMounts: - - name: socket-dir - mountPath: /run/csi - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 - volumeMounts: - - name: socket-dir - mountPath: /run/csi - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 - args: - - --feature-gates=Topology=true - - --default-fstype=ext4 - volumeMounts: - - name: socket-dir - mountPath: /run/csi - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:1.6.0 - imagePullPolicy: Always - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - name: hcloud-csi - key: token - volumeMounts: - - name: socket-dir - mountPath: /run/csi - ports: - - containerPort: 9189 - name: metrics - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - - name: liveness-probe - imagePullPolicy: Always - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 - volumeMounts: - - mountPath: /run/csi - name: socket-dir + - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + name: csi-attacher + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 + name: csi-resizer + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - args: + - --feature-gates=Topology=true + - --default-fstype=ext4 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 + name: csi-provisioner + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hcloud-csi + image: hetznercloud/hcloud-csi-driver:1.6.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: socket-dir + serviceAccount: hcloud-csi volumes: - - name: socket-dir - emptyDir: {} + - emptyDir: {} + name: socket-dir --- -kind: DaemonSet apiVersion: apps/v1 +kind: DaemonSet metadata: - name: hcloud-csi-node - namespace: kube-system labels: app: hcloud-csi + name: hcloud-csi-node + namespace: kube-system spec: selector: matchLabels: @@ -222,136 +327,115 @@ spec: labels: app: hcloud-csi spec: - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: "instance.hetzner.cloud/is-root-server" - operator: NotIn - values: - - "true" - serviceAccount: hcloud-csi + - matchExpressions: + - key: instance.hetzner.cloud/is-root-server + operator: NotIn + values: + - "true" containers: - - name: csi-node-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 - args: - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /run/csi - - name: registration-dir - mountPath: /registration - securityContext: - privileged: true - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:1.6.0 - imagePullPolicy: Always - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - name: hcloud-csi - key: token - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /run/csi - - name: device-dir - mountPath: /dev - securityContext: - privileged: true - ports: - - containerPort: 9189 - name: metrics - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - - name: liveness-probe - imagePullPolicy: Always - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 - volumeMounts: - - mountPath: /run/csi - name: plugin-dir + - args: + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 + name: csi-node-driver-registrar + securityContext: + privileged: true + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hcloud-csi + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: hetznercloud/hcloud-csi-driver:1.6.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /run/csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + serviceAccount: hcloud-csi + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists volumes: - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir --- -apiVersion: v1 -kind: Service +apiVersion: storage.k8s.io/v1 +kind: CSIDriver metadata: - name: hcloud-csi-controller-metrics - namespace: kube-system - labels: - app: hcloud-csi + name: csi.hetzner.cloud spec: - selector: - app: hcloud-csi-controller - ports: - - port: 9189 - name: metrics - targetPort: metrics - ---- -apiVersion: v1 -kind: Service -metadata: - name: hcloud-csi-node-metrics - namespace: kube-system - labels: - app: hcloud-csi -spec: - selector: - app: hcloud-csi - ports: - - port: 9189 - name: metrics - targetPort: metrics + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent From c7afa8a9fc36f7c07954e121b4885c4bbe799c11 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 25 Aug 2022 19:39:26 +0300 Subject: [PATCH 2/3] Update CSI driver to latest for Hetzner --- .../k8s-1.22.yaml.template | 73 +++++-------------- 1 file changed, 17 insertions(+), 56 deletions(-) diff --git a/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template b/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template index e8ff927be9..f0bc615324 100644 --- a/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template +++ b/upup/models/cloudup/resources/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml.template @@ -1,4 +1,4 @@ -# Pulled and modified from: https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml +# Pulled and modified from: https://raw.githubusercontent.com/hetznercloud/csi-driver/main/deploy/kubernetes/hcloud-csi.yml --- apiVersion: v1 kind: Secret @@ -22,13 +22,13 @@ volumeBindingMode: WaitForFirstConsumer apiVersion: v1 kind: ServiceAccount metadata: - name: hcloud-csi + name: hcloud-csi-controller namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: hcloud-csi + name: hcloud-csi-controller rules: - apiGroups: - "" @@ -164,14 +164,14 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: hcloud-csi + name: hcloud-csi-controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: hcloud-csi + name: hcloud-csi-controller subjects: - kind: ServiceAccount - name: hcloud-csi + name: hcloud-csi-controller namespace: kube-system --- apiVersion: v1 @@ -205,7 +205,7 @@ spec: app: hcloud-csi --- apiVersion: apps/v1 -kind: StatefulSet +kind: Deployment metadata: name: hcloud-csi-controller namespace: kube-system @@ -214,7 +214,6 @@ spec: selector: matchLabels: app: hcloud-csi-controller - serviceName: hcloud-csi-controller template: metadata: labels: @@ -223,23 +222,11 @@ spec: containers: - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 name: csi-attacher - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true volumeMounts: - mountPath: /run/csi name: socket-dir - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 name: csi-resizer - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true volumeMounts: - mountPath: /run/csi name: socket-dir @@ -248,16 +235,12 @@ spec: - --default-fstype=ext4 image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 name: csi-provisioner - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true volumeMounts: - mountPath: /run/csi name: socket-dir - - env: + - command: + - /bin/hcloud-csi-driver-controller + env: - name: CSI_ENDPOINT value: unix:///run/csi/socket - name: METRICS_ENDPOINT @@ -274,7 +257,7 @@ spec: secretKeyRef: key: token name: hcloud-csi - image: hetznercloud/hcloud-csi-driver:1.6.0 + image: hetznercloud/hcloud-csi-driver:latest imagePullPolicy: Always livenessProbe: failureThreshold: 5 @@ -291,12 +274,6 @@ spec: - containerPort: 9808 name: healthz protocol: TCP - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true volumeMounts: - mountPath: /run/csi name: socket-dir @@ -306,7 +283,7 @@ spec: volumeMounts: - mountPath: /run/csi name: socket-dir - serviceAccount: hcloud-csi + serviceAccountName: hcloud-csi-controller volumes: - emptyDir: {} name: socket-dir @@ -339,39 +316,23 @@ spec: containers: - args: - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 name: csi-node-driver-registrar - securityContext: - privileged: true volumeMounts: - mountPath: /run/csi name: plugin-dir - mountPath: /registration name: registration-dir - - env: + - command: + - /bin/hcloud-csi-driver-node + env: - name: CSI_ENDPOINT value: unix:///run/csi/socket - name: METRICS_ENDPOINT value: 0.0.0.0:9189 - name: ENABLE_METRICS value: "true" - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud-csi - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - image: hetznercloud/hcloud-csi-driver:1.6.0 + image: hetznercloud/hcloud-csi-driver:latest imagePullPolicy: Always livenessProbe: failureThreshold: 5 @@ -404,7 +365,6 @@ spec: volumeMounts: - mountPath: /run/csi name: plugin-dir - serviceAccount: hcloud-csi tolerations: - effect: NoExecute operator: Exists @@ -436,6 +396,7 @@ metadata: name: csi.hetzner.cloud spec: attachRequired: true + fsGroupPolicy: File podInfoOnMount: true volumeLifecycleModes: - Persistent From ae711d90e19e6e0be3cc8340f1e9e16af941299b Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 25 Aug 2022 19:56:10 +0300 Subject: [PATCH 3/3] Run hack/update-expected.sh --- ...minimal.k8s.local-addons-bootstrap_content | 2 +- ...-csi-driver.addons.k8s.io-k8s-1.22_content | 487 ++++++++---------- 2 files changed, 226 insertions(+), 263 deletions(-) diff --git a/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content b/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content index 5046481329..ad78c89e76 100644 --- a/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content @@ -55,7 +55,7 @@ spec: version: 9.99.0 - id: k8s-1.22 manifest: hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml - manifestHash: ab12002aa9a1c17f7568acc659dd38f73f14c2547ce75dafef0d95a15cb0b189 + manifestHash: 4c3eaaab2359e91bbd50ca60a3f84de376ecbd2e1bab32de4bce758e2184deed name: hcloud-csi-driver.addons.k8s.io selector: k8s-addon: hcloud-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content b/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content index 4937c64c83..75d34fd1ed 100644 --- a/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content +++ b/tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content @@ -13,23 +13,6 @@ stringData: --- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: hcloud-csi-driver.addons.k8s.io - name: csi.hetzner.cloud -spec: - attachRequired: true - podInfoOnMount: true - volumeLifecycleModes: - - Persistent - ---- - allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -56,7 +39,7 @@ metadata: addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io app.kubernetes.io/managed-by: kops k8s-addon: hcloud-csi-driver.addons.k8s.io - name: hcloud-csi + name: hcloud-csi-controller namespace: kube-system --- @@ -69,7 +52,7 @@ metadata: addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io app.kubernetes.io/managed-by: kops k8s-addon: hcloud-csi-driver.addons.k8s.io - name: hcloud-csi + name: hcloud-csi-controller rules: - apiGroups: - "" @@ -212,256 +195,15 @@ metadata: addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io app.kubernetes.io/managed-by: kops k8s-addon: hcloud-csi-driver.addons.k8s.io - name: hcloud-csi + name: hcloud-csi-controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: hcloud-csi + name: hcloud-csi-controller subjects: - kind: ServiceAccount - name: hcloud-csi - namespace: kube-system - ---- - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: hcloud-csi-driver.addons.k8s.io name: hcloud-csi-controller namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app: hcloud-csi-controller - serviceName: hcloud-csi-controller - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 - name: csi-attacher - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 - name: csi-resizer - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - args: - - --feature-gates=Topology=true - - --default-fstype=ext4 - image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 - name: csi-provisioner - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud-csi - image: hetznercloud/hcloud-csi-driver:1.6.0 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: socket-dir - serviceAccount: hcloud-csi - volumes: - - emptyDir: {} - name: socket-dir - ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io - app: hcloud-csi - app.kubernetes.io/managed-by: kops - k8s-addon: hcloud-csi-driver.addons.k8s.io - name: hcloud-csi-node - namespace: kube-system -spec: - selector: - matchLabels: - app: hcloud-csi - template: - metadata: - creationTimestamp: null - labels: - app: hcloud-csi - kops.k8s.io/managed-by: kops - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: instance.hetzner.cloud/is-root-server - operator: NotIn - values: - - "true" - containers: - - args: - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 - name: csi-node-driver-registrar - securityContext: - privileged: true - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud-csi - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - image: hetznercloud/hcloud-csi-driver:1.6.0 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /run/csi - name: plugin-dir - - mountPath: /dev - name: device-dir - - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - serviceAccount: hcloud-csi - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /dev - type: Directory - name: device-dir --- @@ -504,3 +246,224 @@ spec: targetPort: metrics selector: app: hcloud-csi + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: hcloud-csi-driver.addons.k8s.io + name: hcloud-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: hcloud-csi-controller + template: + metadata: + creationTimestamp: null + labels: + app: hcloud-csi-controller + kops.k8s.io/managed-by: kops + spec: + containers: + - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + name: csi-attacher + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 + name: csi-resizer + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - args: + - --feature-gates=Topology=true + - --default-fstype=ext4 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 + name: csi-provisioner + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - command: + - /bin/hcloud-csi-driver-controller + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hcloud-csi + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: socket-dir + serviceAccountName: hcloud-csi-controller + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io + app: hcloud-csi + app.kubernetes.io/managed-by: kops + k8s-addon: hcloud-csi-driver.addons.k8s.io + name: hcloud-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: hcloud-csi + template: + metadata: + creationTimestamp: null + labels: + app: hcloud-csi + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: instance.hetzner.cloud/is-root-server + operator: NotIn + values: + - "true" + containers: + - args: + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 + name: csi-node-driver-registrar + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - command: + - /bin/hcloud-csi-driver-node + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /run/csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: hcloud-csi-driver.addons.k8s.io + name: csi.hetzner.cloud +spec: + attachRequired: true + fsGroupPolicy: File + podInfoOnMount: true + volumeLifecycleModes: + - Persistent