diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go index ca8eec4c5a..e8480182db 100644 --- a/nodeup/pkg/model/networking/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -17,7 +17,9 @@ limitations under the License. package networking import ( + "errors" "fmt" + "os" "path/filepath" "golang.org/x/sys/unix" @@ -50,7 +52,11 @@ func (b *CiliumBuilder) Build(c *fi.ModelBuilderContext) error { } if err := b.buildBPFMount(c); err != nil { - return err + return fmt.Errorf("failed to create bpf mount unit: %w", err) + } + + if err := b.buildCgroup2Mount(c); err != nil { + return fmt.Errorf("failed to create cgroupv2 mount unit: %w", err) } return nil @@ -93,6 +99,52 @@ WantedBy=multi-user.target Name: "sys-fs-bpf.mount", Definition: fi.String(unit), } + service.InitDefaults() + c.AddTask(service) + } + + return nil +} + +func (b *CiliumBuilder) buildCgroup2Mount(c *fi.ModelBuilderContext) error { + + cgroupPath := "/run/cilium/cgroupv2" + + var fsdata unix.Statfs_t + err := unix.Statfs(cgroupPath, &fsdata) + + // If the path does not exist, systemd will create it + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("error checking for /run/cilium/cgroupv2: %v", err) + } + + CGROUP_FS_MAGIC := uint32(0x63677270) + + alreadyMounted := uint32(fsdata.Type) == CGROUP_FS_MAGIC + + if !alreadyMounted { + unit := ` +[Unit] +Description=Cilium Cgroup2 mounts +Documentation=http://docs.cilium.io/ +DefaultDependencies=no +Before=local-fs.target umount.target kubelet.service + +[Mount] +What=cgroup2 +Where=/run/cilium/cgroupv2 +Type=cgroup2 + +[Install] +WantedBy=multi-user.target +` + + service := &nodetasks.Service{ + Name: "run-cilium-cgroupv2.mount", + Definition: fi.String(unit), + SmartRestart: fi.Bool(false), + } + service.InitDefaults() c.AddTask(service) } diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content index b1828e9662..e2d2eabded 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 79105657d58e949d8612de05cf98900e30b0e7fb5c6e2c06a7c35692c91f302e + manifestHash: 1703cd96b5c8d24e70cc30e81b011e9f6392a2df4e3a714bccb03b0a9a824f0e name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content index 94276356f0..8b27984eb0 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -35,7 +35,7 @@ data: bpf-nat-global-max: "524288" bpf-neigh-global-max: "524288" bpf-policy-map-max: "16384" - cgroup-root: /sys/fs/cgroup/unified + cgroup-root: /run/cilium/cgroupv2 cluster-name: default container-runtime: none debug: "false" @@ -494,7 +494,7 @@ spec: volumeMounts: - mountPath: /sys/fs/bpf name: bpf-maps - - mountPath: /sys/fs/cgroup/unified + - mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer name: cilium-cgroup - mountPath: /var/run/cilium @@ -520,8 +520,8 @@ spec: type: DirectoryOrCreate name: cni-path - hostPath: - path: /sys/fs/cgroup/unified - type: DirectoryOrCreate + path: /run/cilium/cgroupv2 + type: Directory name: cilium-cgroup - hostPath: path: /etc/cni/net.d diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content index 2db9d6a850..e91d70d785 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 39ec0f473a3c9479e15718e23b965dd5670502132723c7d2d9377f6b800be453 + manifestHash: 96198c21b885265a89a7374b685d06154fe36741c890b38f932a759073bdc82f name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content index af8820f168..819320a0bb 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -35,7 +35,7 @@ data: bpf-nat-global-max: "524288" bpf-neigh-global-max: "524288" bpf-policy-map-max: "16384" - cgroup-root: /sys/fs/cgroup/unified + cgroup-root: /run/cilium/cgroupv2 cluster-name: default container-runtime: none debug: "false" @@ -494,7 +494,7 @@ spec: volumeMounts: - mountPath: /sys/fs/bpf name: bpf-maps - - mountPath: /sys/fs/cgroup/unified + - mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer name: cilium-cgroup - mountPath: /var/run/cilium @@ -520,8 +520,8 @@ spec: type: DirectoryOrCreate name: cni-path - hostPath: - path: /sys/fs/cgroup/unified - type: DirectoryOrCreate + path: /run/cilium/cgroupv2 + type: Directory name: cilium-cgroup - hostPath: path: /etc/cni/net.d diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content index d94f71b0f6..d026131978 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 6fb92a78a0b2be6a6a21dd312e41be98245f64ad104add14d900de041e41ce41 + manifestHash: b0d700920b53b105c93ca2d6bd1c9ca5fcdab045f8e5b4d88f4893be71752c55 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content index a155b8fcdc..bd01a596e9 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -37,7 +37,7 @@ data: bpf-nat-global-max: "524288" bpf-neigh-global-max: "524288" bpf-policy-map-max: "16384" - cgroup-root: /sys/fs/cgroup/unified + cgroup-root: /run/cilium/cgroupv2 cluster-name: default container-runtime: none debug: "false" @@ -514,7 +514,7 @@ spec: volumeMounts: - mountPath: /sys/fs/bpf name: bpf-maps - - mountPath: /sys/fs/cgroup/unified + - mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer name: cilium-cgroup - mountPath: /var/run/cilium @@ -540,8 +540,8 @@ spec: type: DirectoryOrCreate name: cni-path - hostPath: - path: /sys/fs/cgroup/unified - type: DirectoryOrCreate + path: /run/cilium/cgroupv2 + type: Directory name: cilium-cgroup - hostPath: path: /etc/cni/net.d diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template index c8c8649e63..04f9d60df1 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template @@ -232,7 +232,7 @@ data: # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true) enable-l7-proxy: "{{ .EnableL7Proxy }}" - cgroup-root: /sys/fs/cgroup/unified + cgroup-root: /run/cilium/cgroupv2 {{ if WithDefaultBool .Hubble.Enabled false }} # Enable Hubble gRPC service. @@ -761,7 +761,7 @@ spec: name: bpf-maps mountPropagation: HostToContainer # Required to mount cgroup filesystem from the host to cilium agent pod - - mountPath: /sys/fs/cgroup/unified + - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup mountPropagation: HostToContainer - mountPath: /var/run/cilium @@ -797,8 +797,8 @@ spec: name: cni-path # To keep state between restarts / upgrades for cgroup2 filesystem - hostPath: - path: /sys/fs/cgroup/unified - type: DirectoryOrCreate + path: /run/cilium/cgroupv2 + type: Directory name: cilium-cgroup # To install cilium cni configuration in the host - hostPath: diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template index a16c9cdf15..7068df8034 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template @@ -248,7 +248,7 @@ data: # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true) enable-l7-proxy: "{{ .EnableL7Proxy }}" - cgroup-root: /sys/fs/cgroup/unified + cgroup-root: /run/cilium/cgroupv2 {{ if WithDefaultBool .Hubble.Enabled false }} # Enable Hubble gRPC service. @@ -798,7 +798,7 @@ spec: mountPropagation: HostToContainer {{- end }} # Required to mount cgroup filesystem from the host to cilium agent pod - - mountPath: /sys/fs/cgroup/unified + - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup mountPropagation: HostToContainer - mountPath: /var/run/cilium @@ -834,8 +834,8 @@ spec: name: cni-path # To keep state between restarts / upgrades for cgroup2 filesystem - hostPath: - path: /sys/fs/cgroup/unified - type: DirectoryOrCreate + path: /run/cilium/cgroupv2 + type: Directory name: cilium-cgroup # To install cilium cni configuration in the host - hostPath: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index adb2e9b8c6..8f7fb7ffda 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -53,7 +53,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e + manifestHash: a8676c7ed851ffa9059350f44530cc6a89e79abf6e2395a0655c9ec950e543e2 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml index ed3b267c9d..f53a0864d9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml @@ -59,7 +59,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e + manifestHash: a8676c7ed851ffa9059350f44530cc6a89e79abf6e2395a0655c9ec950e543e2 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml index 976a065ce2..597191dc71 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml @@ -53,7 +53,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e + manifestHash: a8676c7ed851ffa9059350f44530cc6a89e79abf6e2395a0655c9ec950e543e2 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml index 79e6af7951..4d0f7bff4c 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml @@ -65,7 +65,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e + manifestHash: a8676c7ed851ffa9059350f44530cc6a89e79abf6e2395a0655c9ec950e543e2 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml index 9ca0ec7821..e7e4b2fe97 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml @@ -59,7 +59,7 @@ spec: k8s-addon: storage-aws.addons.k8s.io - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e + manifestHash: a8676c7ed851ffa9059350f44530cc6a89e79abf6e2395a0655c9ec950e543e2 name: networking.cilium.io needsRollingUpdate: all selector: