Merge pull request #10419 from bharath-123/task/default-systemd

Default cgroup driver to systemd from k8s 1.20

docs/cluster_spec.md

@@ -949,6 +949,7 @@ spec:
 ### Configuration
 
 It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options.
+Overriding the configuration of containerd has to be done with care as the default config may change with new releases and can lead to incompatibilities.
 
 ```yaml
 spec:
@@ -1195,3 +1196,30 @@ spec:
 ```
 
 which would end up in a drop-in file on all masters and nodes of the cluster.
+
+## cgroupDriver
+
+As of Kubernetes 1.20, kOps will default the cgroup driver of the kubelet and the container runtime to use systemd as the default cgroup driver
+as opposed to cgroup fs.
+
+It is important to ensure that the kubelet and the container runtime are using the same cgroup driver. Below are examples showing
+how to set the cgroup driver for kubelet and the container runtime. 
+
+
+Setting kubelet to use cgroupfs
+```yaml
+spec:
+  kubelet:
+    cgroupDriver: cgroupfs
+```
+
+Setting Docker to use cgroupfs
+```yaml
+spec:
+  docker:
+    execOpt:
+      - native.cgroupdriver=cgroupfs
+```
+
+In the case of containerd, the cgroup-driver is dependant on the cgroup driver of kubelet. To use cgroupfs, just update the 
+cgroupDriver of kubelet to use cgroupfs.

nodeup/pkg/model/docker_test.go

@@ -97,6 +97,10 @@ func TestDockerBuilder_BuildFlags(t *testing.T) {
 			kops.DockerConfig{Bridge: fi.String("br0")},
 			"--bridge=br0",
 		},
+		{
+			kops.DockerConfig{ExecOpt: []string{"native.cgroupdriver=systemd"}},
+			"--exec-opt=native.cgroupdriver=systemd",
+		},
 	}
 
 	for _, g := range grid {

pkg/model/components/containerd.go

@@ -61,6 +61,7 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 			for name, endpoints := range containerd.RegistryMirrors {
 				config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints)
 			}
+			config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc", "runtime_type"}, "io.containerd.runc.v2")
 			containerd.ConfigOverride = fi.String(config.String())
 		}
 

pkg/model/components/docker.go

@@ -73,5 +73,23 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error {
 	// and it is an error to specify the flag twice.
 	docker.Storage = fi.String("overlay2,overlay,aufs")
 
+	// Set systemd as the default cgroup driver in docker from k8s 1.20.
+	if b.IsKubernetesGTE("1.20") && getDockerCgroupDriver(docker.ExecOpt) == "" {
+		docker.ExecOpt = append(docker.ExecOpt, "native.cgroupdriver=systemd")
+	}
+
 	return nil
 }
+
+// checks if cgroup-driver is configured or not for docker or not.
+func getDockerCgroupDriver(execOpts []string) string {
+	for _, value := range execOpts {
+		if value == "native.cgroupdriver=systemd" {
+			return "systemd"
+		} else if value == "native.cgroupdriver=cgroupfs" {
+			return "cgroupfs"
+		}
+	}
+
+	return ""
+}

pkg/model/components/kubelet.go

@@ -213,5 +213,10 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	// Set systemd as the default cgroup driver for kubelet from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && clusterSpec.Kubelet.CgroupDriver == "" {
+		clusterSpec.Kubelet.CgroupDriver = "systemd"
+	}
+
 	return nil
 }

tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml

@@ -144,6 +144,13 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
 
         [plugins."io.containerd.grpc.v1.cri"]
 
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
           [plugins."io.containerd.grpc.v1.cri".registry]
 
             [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
@@ -479,6 +486,13 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
 
         [plugins."io.containerd.grpc.v1.cri"]
 
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
           [plugins."io.containerd.grpc.v1.cri".registry]
 
             [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml

@@ -139,6 +139,17 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
   containerd:
     configOverride: |
       version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
     logLevel: info
     version: 1.4.3
   docker:
@@ -452,6 +463,17 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
   containerd:
     configOverride: |
       version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
     logLevel: info
     version: 1.4.3
   docker: