Do not run CAS on spot instances

docs/releases/1.24-NOTES.md

@@ -83,6 +83,8 @@ In particular, if you are using the snapshot-controller addon, upgrade your clus
 
 * Support for Docker has been removed for Kubernetes 1.24+. See https://kubernetes.io/blog/2020/12/02/dockershim-faq
 
+* Cluster Autoscaler can no longer run on spot instances. This is to avoid cluster autoscaler not being scheduled if the spot instances terminate.
+
 # Required actions
 
 # Deprecations

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: c63ec7a9b56c0e183df9431ce7911a97d59cfb8933e905a354dfb4084eef40a9
+    manifestHash: c20d5158b4c6343a85e02495db2ae251e06f1ada95bfde99b471dc4b8c447092
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -309,6 +309,13 @@ spec:
         k8s-app: cluster-autoscaler
         kops.k8s.io/managed-by: kops
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/spot-worker
+                operator: DoesNotExist
       containers:
       - command:
         - ./cluster-autoscaler

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 2411cb5ac0b5dfda10448cf005c619bae9b6464adaf21a96d5ae804fb5a54529
+    manifestHash: abb81e68a6726627a4b2b0d7baf06e73ea73be8c7965fec9d4ea85424ad0b8b2
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -309,6 +309,13 @@ spec:
         k8s-app: cluster-autoscaler
         kops.k8s.io/managed-by: kops
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/spot-worker
+                operator: DoesNotExist
       containers:
       - command:
         - ./cluster-autoscaler

tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 584ad5f9e360a91c32ef7a824128fe420dc4422a9295b953a0c949dce33d5736
+    manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -309,6 +309,13 @@ spec:
         k8s-app: cluster-autoscaler
         kops.k8s.io/managed-by: kops
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/spot-worker
+                operator: DoesNotExist
       containers:
       - command:
         - ./cluster-autoscaler

upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template

@@ -287,19 +287,23 @@ spec:
         k8s-app: cluster-autoscaler
         app.kubernetes.io/name: "cluster-autoscaler"
     spec:
-      {{ if not UseServiceAccountExternalPermissions }}
       nodeSelector: null
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
+            {{ if not UseServiceAccountExternalPermissions }}
             - matchExpressions:
               - key: node-role.kubernetes.io/control-plane
                 operator: Exists
             - matchExpressions:
               - key: node-role.kubernetes.io/master
                 operator: Exists
-      {{ end }}
+            {{ else }}
+            - matchExpressions:
+              - key: node-role.kubernetes.io/spot-worker
+                operator: DoesNotExist
+            {{ end }}
       priorityClassName: "system-cluster-critical"
       dnsPolicy: "ClusterFirst"
       containers: