etcd-manager: support symlinking versions

This is an easy way for us to signal that certain versions are
compatible with each to etcd-manager, which is otherwise
overly-cautious when it comes to unknown versions.

We extend kops-utils to support the `-t` flag (like cp) to write to a
directory; and the `-s` flag (like cp) to use symlinks.  The syntax
isn't identical to cp, but should be semi-familiar and allows us to
minimize the number of initContainers we use.

cmd/kops-utils-cp/main.go

@@ -17,17 +17,18 @@ limitations under the License.
 package main
 
 import (
+	"flag"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"k8s.io/klog/v2"
 )
 
-func copyFile(source, target string) error {
+func copyFile(source, targetDir string, force bool) error {
-	klog.Infof("Copying source file %q to target directory %q", source, target)
+	klog.Infof("Copying source file %q to target directory %q", source, targetDir)
 
 	sf, err := os.Open(source)
 	if err != nil {
@@ -40,42 +41,107 @@ func copyFile(source, target string) error {
 		return fmt.Errorf("unable to stat source file %q: %w", source, err)
 	}
 
-	fn := filepath.Join(target, filepath.Base(source))
+	destPath := filepath.Join(targetDir, filepath.Base(source))
-	df, err := os.Create(fn)
+
+	if force {
+		if err := os.Remove(destPath); err != nil {
+			if os.IsNotExist(err) {
+				// ignore
+			} else {
+				return fmt.Errorf("error removing file %q (for force): %w", destPath, err)
+			}
+		} else {
+			klog.Infof("removed existing file %q (for force)", destPath)
+		}
+	}
+
+	df, err := os.Create(destPath)
 	if err != nil {
-		return fmt.Errorf("unable to create target file %q: %w", fn, err)
+		return fmt.Errorf("unable to create target file %q: %w", destPath, err)
 	}
 	defer df.Close()
 
 	_, err = io.Copy(df, sf)
 	if err != nil {
-		return fmt.Errorf("unable to copy source file %q contents to target file %q: %w", source, fn, err)
+		return fmt.Errorf("unable to copy source file %q contents to target file %q: %w", source, destPath, err)
 	}
 
 	if err := df.Close(); err != nil {
-		return fmt.Errorf("unable to close target file %q: %w", fn, err)
+		return fmt.Errorf("unable to close target file %q: %w", destPath, err)
 	}
-	if err := os.Chmod(fn, fi.Mode()); err != nil {
+	if err := os.Chmod(destPath, fi.Mode()); err != nil {
-		return fmt.Errorf("unable to change mode of target file %q: %w", fn, err)
+		return fmt.Errorf("unable to change mode of target file %q: %w", destPath, err)
 	}
 
 	return nil
 }
 
+func symlinkFile(oldPath, targetDir string, force bool) error {
+	klog.Infof("symlinking source file %q to target directory %q", oldPath, targetDir)
+
+	newPath := filepath.Join(targetDir, filepath.Base(oldPath))
+	if force {
+		if err := os.Remove(newPath); err != nil {
+			if os.IsNotExist(err) {
+				// ignore
+			} else {
+				return fmt.Errorf("error removing file %q (for force): %w", newPath, err)
+			}
+		} else {
+			klog.Infof("removed existing file %q (for force)", newPath)
+		}
+	}
+	if err := os.Symlink(oldPath, newPath); err != nil {
+		return fmt.Errorf("unable to create symlink from %q -> %q: %w", newPath, oldPath, err)
+	}
+
+	return nil
+}
+
+type stringSliceFlags []string
+
+func (f *stringSliceFlags) String() string {
+	return strings.Join(*f, ",")
+}
+
+func (f *stringSliceFlags) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
 func main() {
-	if len(os.Args) < 3 {
+	// We force (overwrite existing files), so we can be idempotent in case of restart
-		log.Fatal("Usage: kops-utils-cp SOURCE ... TARGET")
+	force := true
+
+	var symlink bool
+	flag.BoolVar(&symlink, "symlink", symlink, "make symbolic link")
+	var targetDirs stringSliceFlags
+	flag.Var(&targetDirs, "target-dir", "copy to directory")
+	var sources stringSliceFlags
+	flag.Var(&sources, "src", "source files to copy")
+
+	flag.Parse()
+
+	if len(sources) == 0 || len(targetDirs) == 0 || len(flag.Args()) != 0 {
+		flag.Usage()
+		os.Exit(1)
 	}
 
-	target := os.Args[len(os.Args)-1]
+	for _, targetDir := range targetDirs {
-
+		if err := os.MkdirAll(targetDir, 0755); err != nil {
-	if err := os.MkdirAll(target, 0755); err != nil {
+			klog.Exitf("unable to create target directory %q: %v", targetDir, err)
-		klog.Exitf("unable to create target directory %q: %v", target, err)
 		}
 
-	for _, src := range os.Args[1 : len(os.Args)-1] {
+		for _, src := range sources {
-		if err := copyFile(src, target); err != nil {
+			if symlink {
-			klog.Exitf("unable to copy source file %q to target directory %q: %v", src, target, err)
+				if err := symlinkFile(src, targetDir, force); err != nil {
+					klog.Exitf("unable to copy source file %q to target directory %q: %v", src, targetDir, err)
+				}
+			} else {
+				if err := copyFile(src, targetDir, force); err != nil {
+					klog.Exitf("unable to copy source file %q to target directory %q: %v", src, targetDir, err)
+				}
+			}
 		}
 	}
 }

pkg/model/components/etcdmanager/model.go

@@ -24,6 +24,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/assets"
@@ -200,18 +201,6 @@ spec:
       name: opt
   hostNetwork: true
   hostPID: true # helps with mounting volumes from inside a container
-  initContainers:
-  - args:
-    - /ko-app/kops-utils-cp
-    - /opt/bin
-    command:
-    - /ko-app/kops-utils-cp
-    image: registry.k8s.io/kops/kops-utils-cp:1.28.0-alpha.1
-    name: kops-utils-cp
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
   volumes:
   - hostPath:
       path: /
@@ -229,6 +218,8 @@ spec:
     emptyDir: {}
 `
 
+const kopsUtilsImage = "registry.k8s.io/kops/kops-utils-cp:1.28.0-alpha.1"
+
 // buildPod creates the pod spec, based on the EtcdClusterSpec
 func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
 	var pod *v1.Pod
@@ -256,33 +247,88 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
 	}
 
 	{
-		for _, etcdVersion := range etcdSupportedVersions() {
+		utilMounts := []v1.VolumeMount{
-			initContainer := v1.Container{
-				Name:    "init-etcd-" + strings.ReplaceAll(etcdVersion, ".", "-"),
-				Image:   etcdSupportedImages[etcdVersion],
-				Command: []string{"/opt/bin/kops-utils-cp"},
-				Args: []string{
-					"/usr/local/bin/etcd",
-					"/usr/local/bin/etcdctl",
-					"/opt/etcd-v" + etcdVersion,
-				},
-				VolumeMounts: []v1.VolumeMount{
 			{
 				MountPath: "/opt",
 				Name:      "opt",
 			},
+		}
+		{
+			initContainer := v1.Container{
+				Name:    "kops-utils-cp",
+				Image:   kopsUtilsImage,
+				Command: []string{"/ko-app/kops-utils-cp"},
+				Args: []string{
+					"--target-dir=/opt/kops-utils/",
+					"--src=/ko-app/kops-utils-cp",
 				},
+				VolumeMounts: utilMounts,
 			}
 			pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer)
 		}
 
-		// Remap all init container images via AssetBuilder
+		symlinkToVersions := sets.NewString()
-		for i, container := range pod.Spec.InitContainers {
+		for _, etcdVersion := range etcdSupportedVersions() {
-			remapped, err := b.AssetBuilder.RemapImage(container.Image)
+			if etcdVersion.SymlinkToVersion != "" {
+				symlinkToVersions.Insert(etcdVersion.SymlinkToVersion)
+				continue
+			}
+
+			initContainer := v1.Container{
+				Name:         "init-etcd-" + strings.ReplaceAll(etcdVersion.Version, ".", "-"),
+				Image:        etcdVersion.Image,
+				Command:      []string{"/opt/kops-utils/kops-utils-cp"},
+				VolumeMounts: utilMounts,
+			}
+
+			initContainer.Args = []string{
+				"--target-dir=/opt/etcd-v" + etcdVersion.Version,
+				"--src=/usr/local/bin/etcd",
+				"--src=/usr/local/bin/etcdctl",
+			}
+
+			pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer)
+		}
+
+		for _, symlinkToVersion := range symlinkToVersions.List() {
+			targetVersions := sets.NewString()
+
+			for _, etcdVersion := range etcdSupportedVersions() {
+				if etcdVersion.SymlinkToVersion == symlinkToVersion {
+					targetVersions.Insert(etcdVersion.Version)
+				}
+			}
+
+			initContainer := v1.Container{
+				Name:         "init-etcd-symlinks-" + strings.ReplaceAll(symlinkToVersion, ".", "-"),
+				Image:        kopsUtilsImage,
+				Command:      []string{"/opt/kops-utils/kops-utils-cp"},
+				VolumeMounts: utilMounts,
+			}
+
+			initContainer.Args = []string{
+				"--symlink",
+			}
+			for _, targetVersion := range targetVersions.List() {
+				initContainer.Args = append(initContainer.Args, "--target-dir=/opt/etcd-v"+targetVersion)
+			}
+			// NOTE: Flags must come before positional arguments
+			initContainer.Args = append(initContainer.Args,
+				"--src=/opt/etcd-v"+symlinkToVersion+"/etcd",
+				"--src=/opt/etcd-v"+symlinkToVersion+"/etcdctl",
+			)
+
+			pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer)
+		}
+
+		// Remap image via AssetBuilder
+		for i := range pod.Spec.InitContainers {
+			initContainer := &pod.Spec.InitContainers[i]
+			remapped, err := b.AssetBuilder.RemapImage(initContainer.Image)
 			if err != nil {
 				return nil, fmt.Errorf("unable to remap init container image %q: %w", container.Image, err)
 			}
-			pod.Spec.InitContainers[i].Image = remapped
+			initContainer.Image = remapped
 		}
 	}
 

pkg/model/components/etcdmanager/options.go

@@ -56,7 +56,11 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error {
 			} else {
 				klog.Warningf("Unsupported etcd version %q detected; please update etcd version.", etcdCluster.Version)
 				klog.Warningf("Use export KOPS_FEATURE_FLAGS=SkipEtcdVersionCheck to override this check.")
-				klog.Warningf("Supported etcd versions: %s", strings.Join(etcdSupportedVersions(), ", "))
+				var versions []string
+				for _, v := range etcdSupportedVersions() {
+					versions = append(versions, v.Version)
+				}
+				klog.Warningf("Supported etcd versions: %s", strings.Join(versions, ", "))
 				return fmt.Errorf("etcd version %q is not supported with etcd-manager, please specify a supported version or remove the value to use the recommended version", etcdCluster.Version)
 			}
 		}
@@ -65,34 +69,41 @@ func (b *EtcdManagerOptionsBuilder) BuildOptions(o interface{}) error {
 	return nil
 }
 
-var etcdSupportedImages = map[string]string{
+// etcdVersion describes how we want to support each etcd version.
-	"3.2.24": "registry.k8s.io/etcd:3.2.24-1",
+type etcdVersion struct {
-	"3.3.10": "registry.k8s.io/etcd:3.3.10-0",
+	Version          string
-	"3.3.17": "registry.k8s.io/etcd:3.3.17-0",
+	Image            string
-	"3.4.3":  "registry.k8s.io/etcd:3.4.3-0",
+	SymlinkToVersion string
-	"3.4.13": "registry.k8s.io/etcd:3.4.13-0",
-	"3.5.0":  "registry.k8s.io/etcd:3.5.0-0",
-	"3.5.1":  "registry.k8s.io/etcd:3.5.1-0",
-	"3.5.3":  "registry.k8s.io/etcd:3.5.3-0",
-	"3.5.4":  "registry.k8s.io/etcd:3.5.4-0",
-	"3.5.6":  "registry.k8s.io/etcd:3.5.6-0",
-	"3.5.7":  "registry.k8s.io/etcd:3.5.7-0",
-	"3.5.9":  "registry.k8s.io/etcd:3.5.9-0",
 }
 
-func etcdSupportedVersions() []string {
+var etcdSupportedImages = []etcdVersion{
-	var versions []string
+	{Version: "3.2.24", Image: "registry.k8s.io/etcd:3.2.24-1"},
-	for etcdVersion := range etcdSupportedImages {
+	{Version: "3.3.10", Image: "registry.k8s.io/etcd:3.3.10-0"},
-		versions = append(versions, etcdVersion)
+	{Version: "3.3.17", Image: "registry.k8s.io/etcd:3.3.17-0"},
+	{Version: "3.4.3", Image: "registry.k8s.io/etcd:3.4.3-0"},
+	{Version: "3.4.13", Image: "registry.k8s.io/etcd:3.4.13-0"},
+	{Version: "3.5.0", Image: "registry.k8s.io/etcd:3.5.0-0"},
+	{Version: "3.5.1", Image: "registry.k8s.io/etcd:3.5.1-0"},
+	{Version: "3.5.3", Image: "registry.k8s.io/etcd:3.5.3-0"},
+	{Version: "3.5.4", Image: "registry.k8s.io/etcd:3.5.4-0"},
+	{Version: "3.5.6", Image: "registry.k8s.io/etcd:3.5.6-0"},
+	{Version: "3.5.7", Image: "registry.k8s.io/etcd:3.5.7-0"},
+	{Version: "3.5.9", Image: "registry.k8s.io/etcd:3.5.9-0"},
 }
-	sort.Strings(versions)
+
+func etcdSupportedVersions() []etcdVersion {
+	var versions []etcdVersion
+	versions = append(versions, etcdSupportedImages...)
+	sort.Slice(versions, func(i, j int) bool { return versions[i].Version < versions[j].Version })
 	return versions
 }
 
 func etcdVersionIsSupported(version string) bool {
 	version = strings.TrimPrefix(version, "v")
-	if _, ok := etcdSupportedImages[version]; ok {
+	for _, etcdVersion := range etcdSupportedImages {
+		if etcdVersion.Version == version {
 			return true
 		}
+	}
 	return false
 }