From 3f99d04ca5fbfb15569384b6eb166011940cb54d Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Sun, 3 Aug 2025 14:19:14 +0300 Subject: [PATCH] etcd-manager: Use image volumes to mount etcd images --- pkg/apis/kops/cluster.go | 24 +++ pkg/model/components/etcdmanager/model.go | 31 ++- pkg/model/components/etcdmanager/options.go | 14 ++ .../etcdmanager/tests/minimal/cluster.yaml | 4 +- .../etcdmanager/tests/minimal/tasks.yaml | 204 ++++++------------ 5 files changed, 135 insertions(+), 142 deletions(-) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index ed19e9e923..8d21ddae8b 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -19,6 +19,7 @@ package kops import ( "fmt" + "github.com/blang/semver/v4" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -947,6 +948,29 @@ func (c *Cluster) InstallCNIAssets() bool { c.Spec.Networking.Cilium == nil } +func (c *Cluster) HasImageVolumesSupport() bool { + // Image Volumes was added to Kubernetes v1.31 + // https://kubernetes.io/blog/2024/08/16/kubernetes-1-31-image-volume-source/ + // Image Volumes graduated to beta in Kubernetes v1.33 + // https://kubernetes.io/blog/2025/04/29/kubernetes-v1-33-image-volume-beta/ + if c.IsKubernetesLT("1.33.0") { + return false + } + if c.Spec.Containerd == nil || c.Spec.Containerd.Version == nil { + return false + } + sv, err := semver.ParseTolerant(*c.Spec.Containerd.Version) + if err != nil { + return false + } + // Image Volumes was released in Containerd v2.1.0 + // https://github.com/containerd/containerd/releases/tag/v2.1.0 + if sv.LT(semver.MustParse("2.1.0")) { + return false + } + return true +} + func (c *Cluster) APIInternalName() string { return "api.internal." + c.ObjectMeta.Name } diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 4c31e70331..76ec2e1607 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -236,7 +236,22 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance } } - { + if b.Cluster.HasImageVolumesSupport() { + for _, etcdVersion := range etcdSupportedVersions() { + if etcdVersion.SymlinkToVersion == "" { + volume := v1.Volume{ + Name: "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-"), + VolumeSource: v1.VolumeSource{ + Image: &v1.ImageVolumeSource{ + Reference: b.AssetBuilder.RemapImage(etcdVersion.Image), + PullPolicy: v1.PullIfNotPresent, + }, + }, + } + pod.Spec.Volumes = append(pod.Spec.Volumes, volume) + } + } + } else { utilMounts := []v1.VolumeMount{ { MountPath: "/opt", @@ -331,6 +346,20 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance // Remap image via AssetBuilder container.Image = b.AssetBuilder.RemapImage(container.Image) + + if b.Cluster.HasImageVolumesSupport() { + for _, etcdVersion := range etcdSupportedVersions() { + volumeMount := v1.VolumeMount{ + MountPath: "/opt/etcd-v" + etcdVersion.Version, + } + if etcdVersion.SymlinkToVersion == "" { + volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.Version, ".", "-") + } else { + volumeMount.Name = "etcd-v" + strings.ReplaceAll(etcdVersion.SymlinkToVersion, ".", "-") + } + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + } } var clientHost string diff --git a/pkg/model/components/etcdmanager/options.go b/pkg/model/components/etcdmanager/options.go index 9cde54b2c5..be192d8ed3 100644 --- a/pkg/model/components/etcdmanager/options.go +++ b/pkg/model/components/etcdmanager/options.go @@ -40,6 +40,20 @@ var _ loader.ClusterOptionsBuilder = &EtcdManagerOptionsBuilder{} func (b *EtcdManagerOptionsBuilder) BuildOptions(o *kops.Cluster) error { clusterSpec := &o.Spec + // Image Volumes will become GA in Kubernetes 1.35 + // https://github.com/kubernetes/enhancements/pull/5450 + if b.ControlPlaneKubernetesVersion().IsLT("1.35.0") && o.HasImageVolumesSupport() { + if clusterSpec.ControlPlaneKubelet == nil { + clusterSpec.ControlPlaneKubelet = &kops.KubeletConfigSpec{} + } + if clusterSpec.ControlPlaneKubelet.FeatureGates == nil { + clusterSpec.ControlPlaneKubelet.FeatureGates = make(map[string]string) + } + if _, found := clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"]; !found { + clusterSpec.ControlPlaneKubelet.FeatureGates["ImageVolume"] = "true" + } + } + for i := range clusterSpec.EtcdClusters { etcdCluster := &clusterSpec.EtcdClusters[i] if etcdCluster.Backups == nil { diff --git a/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml index 9ff9b16dc9..65d1116867 100644 --- a/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml +++ b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml @@ -28,7 +28,9 @@ spec: provider: Manager backups: backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events - kubernetesVersion: v1.21.0 + containerd: + version: 2.1.4 + kubernetesVersion: v1.33.0 masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 networking: diff --git a/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml index 4e2f06309b..7f2bf64487 100644 --- a/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml +++ b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml @@ -102,80 +102,34 @@ Contents: | name: pki - mountPath: /opt name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-21 - mountPath: /var/log/etcd.log name: varlogetcd hostNetwork: true hostPID: true - initContainers: - - args: - - --target-dir=/opt/kops-utils/ - - --src=/ko-app/kops-utils-cp - command: - - /ko-app/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: kops-utils-cp - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --target-dir=/opt/etcd-v3.4.13 - - --src=/usr/local/bin/etcd - - --src=/usr/local/bin/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/etcd:v3.4.13 - name: init-etcd-3-4-13 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --target-dir=/opt/etcd-v3.5.21 - - --src=/usr/local/bin/etcd - - --src=/usr/local/bin/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/etcd:v3.5.21 - name: init-etcd-3-5-21 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --symlink - - --target-dir=/opt/etcd-v3.4.3 - - --src=/opt/etcd-v3.4.13/etcd - - --src=/opt/etcd-v3.4.13/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: init-etcd-symlinks-3-4-13 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --symlink - - --target-dir=/opt/etcd-v3.5.0 - - --target-dir=/opt/etcd-v3.5.1 - - --target-dir=/opt/etcd-v3.5.13 - - --target-dir=/opt/etcd-v3.5.17 - - --target-dir=/opt/etcd-v3.5.3 - - --target-dir=/opt/etcd-v3.5.4 - - --target-dir=/opt/etcd-v3.5.6 - - --target-dir=/opt/etcd-v3.5.7 - - --target-dir=/opt/etcd-v3.5.9 - - --src=/opt/etcd-v3.5.21/etcd - - --src=/opt/etcd-v3.5.21/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: init-etcd-symlinks-3-5-21 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt priorityClassName: system-cluster-critical tolerations: - key: CriticalAddonsOnly @@ -195,6 +149,14 @@ Contents: | name: pki - emptyDir: {} name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.21 + name: etcd-v3-5-21 - hostPath: path: /var/log/etcd-events.log type: FileOrCreate @@ -244,80 +206,34 @@ Contents: | name: pki - mountPath: /opt name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-21 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-21 - mountPath: /var/log/etcd.log name: varlogetcd hostNetwork: true hostPID: true - initContainers: - - args: - - --target-dir=/opt/kops-utils/ - - --src=/ko-app/kops-utils-cp - command: - - /ko-app/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: kops-utils-cp - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --target-dir=/opt/etcd-v3.4.13 - - --src=/usr/local/bin/etcd - - --src=/usr/local/bin/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/etcd:v3.4.13 - name: init-etcd-3-4-13 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --target-dir=/opt/etcd-v3.5.21 - - --src=/usr/local/bin/etcd - - --src=/usr/local/bin/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/etcd:v3.5.21 - name: init-etcd-3-5-21 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --symlink - - --target-dir=/opt/etcd-v3.4.3 - - --src=/opt/etcd-v3.4.13/etcd - - --src=/opt/etcd-v3.4.13/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: init-etcd-symlinks-3-4-13 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt - - args: - - --symlink - - --target-dir=/opt/etcd-v3.5.0 - - --target-dir=/opt/etcd-v3.5.1 - - --target-dir=/opt/etcd-v3.5.13 - - --target-dir=/opt/etcd-v3.5.17 - - --target-dir=/opt/etcd-v3.5.3 - - --target-dir=/opt/etcd-v3.5.4 - - --target-dir=/opt/etcd-v3.5.6 - - --target-dir=/opt/etcd-v3.5.7 - - --target-dir=/opt/etcd-v3.5.9 - - --src=/opt/etcd-v3.5.21/etcd - - --src=/opt/etcd-v3.5.21/etcdctl - command: - - /opt/kops-utils/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.34.0-alpha.1 - name: init-etcd-symlinks-3-5-21 - resources: {} - volumeMounts: - - mountPath: /opt - name: opt priorityClassName: system-cluster-critical tolerations: - key: CriticalAddonsOnly @@ -337,6 +253,14 @@ Contents: | name: pki - emptyDir: {} name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.21 + name: etcd-v3-5-21 - hostPath: path: /var/log/etcd.log type: FileOrCreate