From 74fdf9c7a3e0b1bf4d8ab4eac7cfd747871bda4a Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 21 Jun 2022 15:00:31 +0200 Subject: [PATCH] Do not run CAS on spot instances --- docs/releases/1.24-NOTES.md | 2 ++ ...s3_object_minimal.example.com-addons-bootstrap_content | 2 +- ...dons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content | 7 +++++++ ...s3_object_minimal.example.com-addons-bootstrap_content | 2 +- ...dons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content | 7 +++++++ ...s3_object_minimal.example.com-addons-bootstrap_content | 2 +- ...dons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content | 7 +++++++ .../k8s-1.15.yaml.template | 8 ++++++-- 8 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/releases/1.24-NOTES.md b/docs/releases/1.24-NOTES.md index 38f071b929..a08bfad346 100644 --- a/docs/releases/1.24-NOTES.md +++ b/docs/releases/1.24-NOTES.md @@ -83,6 +83,8 @@ In particular, if you are using the snapshot-controller addon, upgrade your clus * Support for Docker has been removed for Kubernetes 1.24+. See https://kubernetes.io/blog/2020/12/02/dockershim-faq +* Cluster Autoscaler can no longer run on spot instances. This is to avoid cluster autoscaler not being scheduled if the spot instances terminate. + # Required actions # Deprecations diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 29a270b226..46ddf55a30 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: c63ec7a9b56c0e183df9431ce7911a97d59cfb8933e905a354dfb4084eef40a9 + manifestHash: c20d5158b4c6343a85e02495db2ae251e06f1ada95bfde99b471dc4b8c447092 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 0125c21280..7671f9de29 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -309,6 +309,13 @@ spec: k8s-app: cluster-autoscaler kops.k8s.io/managed-by: kops spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/spot-worker + operator: DoesNotExist containers: - command: - ./cluster-autoscaler diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index f7aec2714e..4b28d4cec4 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 2411cb5ac0b5dfda10448cf005c619bae9b6464adaf21a96d5ae804fb5a54529 + manifestHash: abb81e68a6726627a4b2b0d7baf06e73ea73be8c7965fec9d4ea85424ad0b8b2 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index e0371f98e7..d0b22e1bdd 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -309,6 +309,13 @@ spec: k8s-app: cluster-autoscaler kops.k8s.io/managed-by: kops spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/spot-worker + operator: DoesNotExist containers: - command: - ./cluster-autoscaler diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 0de348769c..dc3f9bc601 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 584ad5f9e360a91c32ef7a824128fe420dc4422a9295b953a0c949dce33d5736 + manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 0328d3d618..7a2cd09179 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -309,6 +309,13 @@ spec: k8s-app: cluster-autoscaler kops.k8s.io/managed-by: kops spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/spot-worker + operator: DoesNotExist containers: - command: - ./cluster-autoscaler diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template index 78b8c72bd1..ce5e1c0221 100644 --- a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template @@ -287,19 +287,23 @@ spec: k8s-app: cluster-autoscaler app.kubernetes.io/name: "cluster-autoscaler" spec: - {{ if not UseServiceAccountExternalPermissions }} nodeSelector: null affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: + {{ if not UseServiceAccountExternalPermissions }} - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists - {{ end }} + {{ else }} + - matchExpressions: + - key: node-role.kubernetes.io/spot-worker + operator: DoesNotExist + {{ end }} priorityClassName: "system-cluster-critical" dnsPolicy: "ClusterFirst" containers: