Enable etcd corruption check as mitigatio of 3.5 corruption issue

pkg/model/components/etcdmanager/options.go

@@ -49,7 +49,9 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error {
 			etcdCluster.Backups.BackupStore = urls.Join(base, "backups", "etcd", etcdCluster.Name)
 		}
 
-		if !etcdVersionIsSupported(etcdCluster.Version) {
+		version := strings.TrimPrefix(etcdCluster.Version, "v")
+
+		if !etcdVersionIsSupported(version) {
 			if featureflag.SkipEtcdVersionCheck.Enabled() {
 				klog.Warningf("etcd version %q is not known to be supported, but ignoring because of SkipEtcdVersionCheck feature flag", etcdCluster.Version)
 			} else {
@@ -57,15 +59,19 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error {
 				return fmt.Errorf("etcd version %q is not supported with etcd-manager, please specify a supported version or remove the value to use the default version.  Supported versions: %s", etcdCluster.Version, strings.Join(supportedEtcdVersions, ", "))
 			}
 		}
+		for _, s := range []string{"3.5.0", "3.5.1"} {
+			if s == version {
+				appendCorruptionCheckFlag(etcdCluster)
+			}
 		}
 
+	}
 	return nil
 }
 
 var supportedEtcdVersions = []string{"3.1.12", "3.2.18", "3.2.24", "3.3.10", "3.3.13", "3.3.17", "3.4.3", "3.4.13", "3.5.0", "3.5.1"}
 
 func etcdVersionIsSupported(version string) bool {
-	version = strings.TrimPrefix(version, "v")
 	for _, v := range supportedEtcdVersions {
 		if v == version {
 			return true
@@ -73,3 +79,20 @@ func etcdVersionIsSupported(version string) bool {
 	}
 	return false
 }
+
+func appendCorruptionCheckFlag(etcdCluster *kops.EtcdClusterSpec) {
+	varName := "ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK"
+	if etcdCluster.Manager == nil {
+		etcdCluster.Manager = &kops.EtcdManagerSpec{}
+	}
+	for _, env := range etcdCluster.Manager.Env {
+		if env.Name == varName {
+			return
+		}
+	}
+	etcdCluster.Manager.Env = append(etcdCluster.Manager.Env,
+		kops.EnvVar{
+			Name:  varName,
+			Value: "true",
+		})
+}

tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml

@@ -314,8 +314,16 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   encryptionConfig: null
   etcdClusters:
     events:
+      manager:
+        env:
+        - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+          value: "true"
       version: 3.5.1
     main:
+      manager:
+        env:
+        - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+          value: "true"
       version: 3.5.1
   kubeAPIServer:
     allowPrivileged: true

tests/integration/update_cluster/apiservernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -137,8 +137,16 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
   main:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
 kubeAPIServer:
   allowPrivileged: true

tests/integration/update_cluster/apiservernodes/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -31,6 +31,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: main
     version: 3.5.1
   - backups:
@@ -38,6 +42,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: events
     version: 3.5.1
   externalDns:

tests/integration/update_cluster/apiservernodes/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -21,6 +21,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/apiservernodes/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -21,6 +21,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -137,8 +137,16 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
   main:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
 kubeAPIServer:
   allowPrivileged: true

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -56,6 +56,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: main
     version: 3.5.1
   - backups:
@@ -63,6 +67,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: events
     version: 3.5.1
   externalDns:

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -18,6 +18,9 @@ spec:
       --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s.io/etcd/events
       --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -18,6 +18,9 @@ spec:
       --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s.io/etcd/main
       --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-1.23/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -138,10 +138,18 @@ encryptionConfig: null
 etcdClusters:
   events:
     cpuRequest: 100m
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     version: 3.5.1
   main:
     cpuRequest: 200m
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     version: 3.5.1
 kubeAPIServer:

tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -33,6 +33,10 @@ spec:
     - encryptedVolume: true
       instanceGroup: master-us-test-1a
       name: a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     name: main
     version: 3.5.1
@@ -43,6 +47,10 @@ spec:
     - encryptedVolume: true
       instanceGroup: master-us-test-1a
       name: a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     name: events
     version: 3.5.1

tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-1.24/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -138,10 +138,18 @@ encryptionConfig: null
 etcdClusters:
   events:
     cpuRequest: 100m
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     version: 3.5.1
   main:
     cpuRequest: 200m
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     version: 3.5.1
 kubeAPIServer:

tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -42,6 +42,10 @@ spec:
     - encryptedVolume: true
       instanceGroup: master-us-test-1a
       name: a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     name: main
     version: 3.5.1
@@ -52,6 +56,10 @@ spec:
     - encryptedVolume: true
       instanceGroup: master-us-test-1a
       name: a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     memoryRequest: 100Mi
     name: events
     version: 3.5.1

tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main
       --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned
       > /tmp/pipe 2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data

@@ -140,8 +140,16 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
   main:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
 kubeAPIServer:
   allowPrivileged: true

tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -44,6 +44,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: main
     version: 3.5.1
   - backups:
@@ -51,6 +55,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: events
     version: 3.5.1
   externalDns:

tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -19,6 +19,9 @@ spec:
       --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/minimal-ipv6.example.com=owned > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -19,6 +19,9 @@ spec:
       --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/minimal-ipv6.example.com=owned > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -32,6 +32,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test1-a
       name: "1"
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: main
     version: 3.5.1
   - backups:
@@ -39,6 +43,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test1-a
       name: "1"
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: events
     version: 3.5.1
   externalDns:

tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s-io-etcd-events --volume-provider=gce --volume-tag=k8s-io-cluster-name=minimal-gce-example-com
       --volume-tag=k8s-io-etcd-events --volume-tag=k8s-io-role-master=master > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -18,6 +18,9 @@ spec:
       --v=6 --volume-name-tag=k8s-io-etcd-main --volume-provider=gce --volume-tag=k8s-io-cluster-name=minimal-gce-example-com
       --volume-tag=k8s-io-etcd-main --volume-tag=k8s-io-role-master=master > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script

@@ -137,8 +137,16 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
   main:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
 kubeAPIServer:
   allowPrivileged: true

tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data

@@ -137,8 +137,16 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
   main:
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     version: 3.5.1
 kubeAPIServer:
   allowPrivileged: true

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -33,6 +33,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: main
     version: 3.5.1
   - backups:
@@ -40,6 +44,10 @@ spec:
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
+    manager:
+      env:
+      - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+        value: "true"
     name: events
     version: 3.5.1
   externalDns:

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_manifests-etcdmanager-events_content

@@ -19,6 +19,9 @@ spec:
       --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/privatecanal.example.com=owned > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources:

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_manifests-etcdmanager-main_content

@@ -19,6 +19,9 @@ spec:
       --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1
       --volume-tag=kubernetes.io/cluster/privatecanal.example.com=owned > /tmp/pipe
       2>&1
+    env:
+    - name: ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK
+      value: "true"
     image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220203
     name: etcd-manager
     resources: