From d6350a5a6e9b81401f395bfe3b016e4d19001dd4 Mon Sep 17 00:00:00 2001 From: justinsb Date: Sat, 17 Jun 2023 21:59:49 -0400 Subject: [PATCH] etcd-manager: support symlinking versions This is an easy way for us to signal that certain versions are compatible with each to etcd-manager, which is otherwise overly-cautious when it comes to unknown versions. We extend kops-utils to support the `-t` flag (like cp) to write to a directory; and the `-s` flag (like cp) to use symlinks. The syntax isn't identical to cp, but should be semi-familiar and allows us to minimize the number of initContainers we use. --- cmd/kops-utils-cp/main.go | 106 ++++++++++++++++---- pkg/model/components/etcdmanager/model.go | 104 +++++++++++++------ pkg/model/components/etcdmanager/options.go | 55 ++++++---- 3 files changed, 194 insertions(+), 71 deletions(-) diff --git a/cmd/kops-utils-cp/main.go b/cmd/kops-utils-cp/main.go index ff641a454d..75e7bca75c 100644 --- a/cmd/kops-utils-cp/main.go +++ b/cmd/kops-utils-cp/main.go @@ -17,17 +17,18 @@ limitations under the License. package main import ( + "flag" "fmt" "io" - "log" "os" "path/filepath" + "strings" "k8s.io/klog/v2" ) -func copyFile(source, target string) error { - klog.Infof("Copying source file %q to target directory %q", source, target) +func copyFile(source, targetDir string, force bool) error { + klog.Infof("Copying source file %q to target directory %q", source, targetDir) sf, err := os.Open(source) if err != nil { @@ -40,42 +41,107 @@ func copyFile(source, target string) error { return fmt.Errorf("unable to stat source file %q: %w", source, err) } - fn := filepath.Join(target, filepath.Base(source)) - df, err := os.Create(fn) + destPath := filepath.Join(targetDir, filepath.Base(source)) + + if force { + if err := os.Remove(destPath); err != nil { + if os.IsNotExist(err) { + // ignore + } else { + return fmt.Errorf("error removing file %q (for force): %w", destPath, err) + } + } else { + klog.Infof("removed existing file %q (for force)", destPath) + } + } + + df, err := os.Create(destPath) if err != nil { - return fmt.Errorf("unable to create target file %q: %w", fn, err) + return fmt.Errorf("unable to create target file %q: %w", destPath, err) } defer df.Close() _, err = io.Copy(df, sf) if err != nil { - return fmt.Errorf("unable to copy source file %q contents to target file %q: %w", source, fn, err) + return fmt.Errorf("unable to copy source file %q contents to target file %q: %w", source, destPath, err) } if err := df.Close(); err != nil { - return fmt.Errorf("unable to close target file %q: %w", fn, err) + return fmt.Errorf("unable to close target file %q: %w", destPath, err) } - if err := os.Chmod(fn, fi.Mode()); err != nil { - return fmt.Errorf("unable to change mode of target file %q: %w", fn, err) + if err := os.Chmod(destPath, fi.Mode()); err != nil { + return fmt.Errorf("unable to change mode of target file %q: %w", destPath, err) } return nil } +func symlinkFile(oldPath, targetDir string, force bool) error { + klog.Infof("symlinking source file %q to target directory %q", oldPath, targetDir) + + newPath := filepath.Join(targetDir, filepath.Base(oldPath)) + if force { + if err := os.Remove(newPath); err != nil { + if os.IsNotExist(err) { + // ignore + } else { + return fmt.Errorf("error removing file %q (for force): %w", newPath, err) + } + } else { + klog.Infof("removed existing file %q (for force)", newPath) + } + } + if err := os.Symlink(oldPath, newPath); err != nil { + return fmt.Errorf("unable to create symlink from %q -> %q: %w", newPath, oldPath, err) + } + + return nil +} + +type stringSliceFlags []string + +func (f *stringSliceFlags) String() string { + return strings.Join(*f, ",") +} + +func (f *stringSliceFlags) Set(value string) error { + *f = append(*f, value) + return nil +} + func main() { - if len(os.Args) < 3 { - log.Fatal("Usage: kops-utils-cp SOURCE ... TARGET") + // We force (overwrite existing files), so we can be idempotent in case of restart + force := true + + var symlink bool + flag.BoolVar(&symlink, "symlink", symlink, "make symbolic link") + var targetDirs stringSliceFlags + flag.Var(&targetDirs, "target-dir", "copy to directory") + var sources stringSliceFlags + flag.Var(&sources, "src", "source files to copy") + + flag.Parse() + + if len(sources) == 0 || len(targetDirs) == 0 || len(flag.Args()) != 0 { + flag.Usage() + os.Exit(1) } - target := os.Args[len(os.Args)-1] + for _, targetDir := range targetDirs { + if err := os.MkdirAll(targetDir, 0755); err != nil { + klog.Exitf("unable to create target directory %q: %v", targetDir, err) + } - if err := os.MkdirAll(target, 0755); err != nil { - klog.Exitf("unable to create target directory %q: %v", target, err) - } - - for _, src := range os.Args[1 : len(os.Args)-1] { - if err := copyFile(src, target); err != nil { - klog.Exitf("unable to copy source file %q to target directory %q: %v", src, target, err) + for _, src := range sources { + if symlink { + if err := symlinkFile(src, targetDir, force); err != nil { + klog.Exitf("unable to copy source file %q to target directory %q: %v", src, targetDir, err) + } + } else { + if err := copyFile(src, targetDir, force); err != nil { + klog.Exitf("unable to copy source file %q to target directory %q: %v", src, targetDir, err) + } + } } } } diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 0839754cca..5fab2e5122 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" @@ -200,18 +201,6 @@ spec: name: opt hostNetwork: true hostPID: true # helps with mounting volumes from inside a container - initContainers: - - args: - - /ko-app/kops-utils-cp - - /opt/bin - command: - - /ko-app/kops-utils-cp - image: registry.k8s.io/kops/kops-utils-cp:1.28.0-alpha.1 - name: kops-utils-cp - resources: {} - volumeMounts: - - mountPath: /opt - name: opt volumes: - hostPath: path: / @@ -229,6 +218,8 @@ spec: emptyDir: {} ` +const kopsUtilsImage = "registry.k8s.io/kops/kops-utils-cp:1.28.0-alpha.1" + // buildPod creates the pod spec, based on the EtcdClusterSpec func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) { var pod *v1.Pod @@ -256,33 +247,88 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance } { - for _, etcdVersion := range etcdSupportedVersions() { + utilMounts := []v1.VolumeMount{ + { + MountPath: "/opt", + Name: "opt", + }, + } + { initContainer := v1.Container{ - Name: "init-etcd-" + strings.ReplaceAll(etcdVersion, ".", "-"), - Image: etcdSupportedImages[etcdVersion], - Command: []string{"/opt/bin/kops-utils-cp"}, + Name: "kops-utils-cp", + Image: kopsUtilsImage, + Command: []string{"/ko-app/kops-utils-cp"}, Args: []string{ - "/usr/local/bin/etcd", - "/usr/local/bin/etcdctl", - "/opt/etcd-v" + etcdVersion, - }, - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "/opt", - Name: "opt", - }, + "--target-dir=/opt/kops-utils/", + "--src=/ko-app/kops-utils-cp", }, + VolumeMounts: utilMounts, } pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) } - // Remap all init container images via AssetBuilder - for i, container := range pod.Spec.InitContainers { - remapped, err := b.AssetBuilder.RemapImage(container.Image) + symlinkToVersions := sets.NewString() + for _, etcdVersion := range etcdSupportedVersions() { + if etcdVersion.SymlinkToVersion != "" { + symlinkToVersions.Insert(etcdVersion.SymlinkToVersion) + continue + } + + initContainer := v1.Container{ + Name: "init-etcd-" + strings.ReplaceAll(etcdVersion.Version, ".", "-"), + Image: etcdVersion.Image, + Command: []string{"/opt/kops-utils/kops-utils-cp"}, + VolumeMounts: utilMounts, + } + + initContainer.Args = []string{ + "--target-dir=/opt/etcd-v" + etcdVersion.Version, + "--src=/usr/local/bin/etcd", + "--src=/usr/local/bin/etcdctl", + } + + pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + } + + for _, symlinkToVersion := range symlinkToVersions.List() { + targetVersions := sets.NewString() + + for _, etcdVersion := range etcdSupportedVersions() { + if etcdVersion.SymlinkToVersion == symlinkToVersion { + targetVersions.Insert(etcdVersion.Version) + } + } + + initContainer := v1.Container{ + Name: "init-etcd-symlinks-" + strings.ReplaceAll(symlinkToVersion, ".", "-"), + Image: kopsUtilsImage, + Command: []string{"/opt/kops-utils/kops-utils-cp"}, + VolumeMounts: utilMounts, + } + + initContainer.Args = []string{ + "--symlink", + } + for _, targetVersion := range targetVersions.List() { + initContainer.Args = append(initContainer.Args, "--target-dir=/opt/etcd-v"+targetVersion) + } + // NOTE: Flags must come before positional arguments + initContainer.Args = append(initContainer.Args, + "--src=/opt/etcd-v"+symlinkToVersion+"/etcd", + "--src=/opt/etcd-v"+symlinkToVersion+"/etcdctl", + ) + + pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + } + + // Remap image via AssetBuilder + for i := range pod.Spec.InitContainers { + initContainer := &pod.Spec.InitContainers[i] + remapped, err := b.AssetBuilder.RemapImage(initContainer.Image) if err != nil { return nil, fmt.Errorf("unable to remap init container image %q: %w", container.Image, err) } - pod.Spec.InitContainers[i].Image = remapped + initContainer.Image = remapped } } diff --git a/pkg/model/components/etcdmanager/options.go b/pkg/model/components/etcdmanager/options.go index 117620b5df..2006222a13 100644 --- a/pkg/model/components/etcdmanager/options.go +++ b/pkg/model/components/etcdmanager/options.go @@ -56,7 +56,11 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error { } else { klog.Warningf("Unsupported etcd version %q detected; please update etcd version.", etcdCluster.Version) klog.Warningf("Use export KOPS_FEATURE_FLAGS=SkipEtcdVersionCheck to override this check.") - klog.Warningf("Supported etcd versions: %s", strings.Join(etcdSupportedVersions(), ", ")) + var versions []string + for _, v := range etcdSupportedVersions() { + versions = append(versions, v.Version) + } + klog.Warningf("Supported etcd versions: %s", strings.Join(versions, ", ")) return fmt.Errorf("etcd version %q is not supported with etcd-manager, please specify a supported version or remove the value to use the recommended version", etcdCluster.Version) } } @@ -65,34 +69,41 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error { return nil } -var etcdSupportedImages = map[string]string{ - "3.2.24": "registry.k8s.io/etcd:3.2.24-1", - "3.3.10": "registry.k8s.io/etcd:3.3.10-0", - "3.3.17": "registry.k8s.io/etcd:3.3.17-0", - "3.4.3": "registry.k8s.io/etcd:3.4.3-0", - "3.4.13": "registry.k8s.io/etcd:3.4.13-0", - "3.5.0": "registry.k8s.io/etcd:3.5.0-0", - "3.5.1": "registry.k8s.io/etcd:3.5.1-0", - "3.5.3": "registry.k8s.io/etcd:3.5.3-0", - "3.5.4": "registry.k8s.io/etcd:3.5.4-0", - "3.5.6": "registry.k8s.io/etcd:3.5.6-0", - "3.5.7": "registry.k8s.io/etcd:3.5.7-0", - "3.5.9": "registry.k8s.io/etcd:3.5.9-0", +// etcdVersion describes how we want to support each etcd version. +type etcdVersion struct { + Version string + Image string + SymlinkToVersion string } -func etcdSupportedVersions() []string { - var versions []string - for etcdVersion := range etcdSupportedImages { - versions = append(versions, etcdVersion) - } - sort.Strings(versions) +var etcdSupportedImages = []etcdVersion{ + {Version: "3.2.24", Image: "registry.k8s.io/etcd:3.2.24-1"}, + {Version: "3.3.10", Image: "registry.k8s.io/etcd:3.3.10-0"}, + {Version: "3.3.17", Image: "registry.k8s.io/etcd:3.3.17-0"}, + {Version: "3.4.3", Image: "registry.k8s.io/etcd:3.4.3-0"}, + {Version: "3.4.13", Image: "registry.k8s.io/etcd:3.4.13-0"}, + {Version: "3.5.0", Image: "registry.k8s.io/etcd:3.5.0-0"}, + {Version: "3.5.1", Image: "registry.k8s.io/etcd:3.5.1-0"}, + {Version: "3.5.3", Image: "registry.k8s.io/etcd:3.5.3-0"}, + {Version: "3.5.4", Image: "registry.k8s.io/etcd:3.5.4-0"}, + {Version: "3.5.6", Image: "registry.k8s.io/etcd:3.5.6-0"}, + {Version: "3.5.7", Image: "registry.k8s.io/etcd:3.5.7-0"}, + {Version: "3.5.9", Image: "registry.k8s.io/etcd:3.5.9-0"}, +} + +func etcdSupportedVersions() []etcdVersion { + var versions []etcdVersion + versions = append(versions, etcdSupportedImages...) + sort.Slice(versions, func(i, j int) bool { return versions[i].Version < versions[j].Version }) return versions } func etcdVersionIsSupported(version string) bool { version = strings.TrimPrefix(version, "v") - if _, ok := etcdSupportedImages[version]; ok { - return true + for _, etcdVersion := range etcdSupportedImages { + if etcdVersion.Version == version { + return true + } } return false }