From beb5cc5ac508290feb561f312170b7d2ad958d33 Mon Sep 17 00:00:00 2001 From: justinsb Date: Thu, 20 Feb 2025 09:07:13 -0500 Subject: [PATCH] metal: copy control plane config to nodes This avoids the needs for these nodes to have access to the state store. * We no longer need S3/GCS credentials on the node. * We don't depend on S3/GCS being reachable from the control plane - at least here! --- pkg/commands/toolbox_enroll.go | 127 ++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 11 deletions(-) diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go index 1b3af073a4..268563d6a4 100644 --- a/pkg/commands/toolbox_enroll.go +++ b/pkg/commands/toolbox_enroll.go @@ -28,7 +28,6 @@ import ( "net" "os" "path" - "path/filepath" "sort" "strconv" "strings" @@ -139,15 +138,15 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr sudo = false } - host, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) + sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) if err != nil { return err } - defer host.Close() + defer sshTarget.Close() publicKeyPath := "/etc/kubernetes/kops/pki/machine/public.pem" - publicKeyBytes, err := host.readFile(ctx, publicKeyPath) + publicKeyBytes, err := sshTarget.readFile(ctx, publicKeyPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { publicKeyBytes = nil @@ -158,11 +157,11 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr publicKeyBytes = bytes.TrimSpace(publicKeyBytes) if len(publicKeyBytes) == 0 { - if _, err := host.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil { + if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil { return err } - b, err := host.readFile(ctx, publicKeyPath) + b, err := sshTarget.readFile(ctx, publicKeyPath) if err != nil { return fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err) } @@ -170,7 +169,7 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr } klog.Infof("public key is %s", string(publicKeyBytes)) - hostname, err := host.getHostname(ctx) + hostname, err := sshTarget.getHostname(ctx) if err != nil { return err } @@ -184,13 +183,13 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr } for k, v := range bootstrapData.NodeupScriptAdditionalFiles { - if err := host.writeFile(ctx, k, bytes.NewReader(v)); err != nil { + if err := sshTarget.writeFile(ctx, k, bytes.NewReader(v)); err != nil { return fmt.Errorf("writing file %q over SSH: %w", k, err) } } if len(bootstrapData.NodeupScript) != 0 { - if _, err := host.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil { + if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil { return err } } @@ -800,7 +799,113 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e return nil, err } + vfsContext := clientset.VFSContext() + + // If this is the control plane, we want to copy the config from s3/gcs to the local file system on the target node, + // so that we don't need credentials to the state store. if bootConfig.InstanceGroupRole == kops.InstanceGroupRoleControlPlane { + remapPrefix := "s3://" // TODO: Support GCS? + + // targetDir is the location of the config on the target node. + targetDir := "/etc/kubernetes/kops/config" + + // remapFile remaps a file from s3/gcs etc to the local file system on the target node. + remapFile := func(pSrc *string, destDir string) error { + src := *pSrc + if !strings.HasPrefix(src, remapPrefix) { + return nil + } + + srcPath, err := vfsContext.BuildVfsPath(src) + if err != nil { + return fmt.Errorf("building vfs path: %w", err) + } + b, err := srcPath.ReadFile(ctx) + if err != nil { + return fmt.Errorf("reading file: %w", err) + } + + dest := strings.TrimPrefix(src, remapPrefix) + dest = path.Join(destDir, dest) + bootstrapData.NodeupScriptAdditionalFiles[dest] = b + + *pSrc = dest + return nil + } + + // remapTree remaps a file tree from s3/gcs etc to the local file system on the target node. + remapTree := func(pSrc *string, dest string) error { + src := *pSrc + if !strings.HasPrefix(src, remapPrefix) { + return nil + } + + srcPath, err := vfsContext.BuildVfsPath(src) + if err != nil { + return fmt.Errorf("building vfs path: %w", err) + } + + srcFiles, err := srcPath.ReadTree(ctx) + if err != nil { + return fmt.Errorf("reading tree: %w", err) + } + basePath := srcPath.Path() + for _, srcFile := range srcFiles { + b, err := srcFile.ReadFile(ctx) + if err != nil { + return fmt.Errorf("reading file: %w", err) + } + + if !strings.HasPrefix(srcFile.Path(), basePath) { + return fmt.Errorf("unexpected path: %q", srcFile.Path()) + } + relativePath := strings.TrimPrefix(srcFile.Path(), basePath) + + bootstrapData.NodeupScriptAdditionalFiles[path.Join(dest, relativePath)] = b + } + + *pSrc = dest + return nil + } + + for i := range nodeupConfig.EtcdManifests { + if err := remapFile(&nodeupConfig.EtcdManifests[i], path.Join(targetDir)); err != nil { + return nil, err + } + } + + configBase, err := vfs.Context.BuildVfsPath(cluster.Spec.ConfigStore.Base) + if err != nil { + return nil, fmt.Errorf("parsing configStore.base %q: %w", cluster.Spec.ConfigStore.Base, err) + } + for i, channel := range nodeupConfig.Channels { + bootstrapChannelPath := configBase.Join("addons", "bootstrap-channel.yaml").Path() + if channel != bootstrapChannelPath { + klog.Infof("not remapping non-bootstrap channel %q", channel) + continue + } + + parentPath := configBase.Join("addons").Path() + if err := remapTree(&parentPath, path.Join(targetDir, "addons")); err != nil { + return nil, err + } + nodeupConfig.Channels[i] = path.Join(parentPath, "bootstrap-channel.yaml") + + // The channels tool requires a file:// prefix + if strings.HasPrefix(nodeupConfig.Channels[i], "/") { + nodeupConfig.Channels[i] = "file://" + nodeupConfig.Channels[i] + } + } + + if nodeupConfig.ConfigStore != nil { + if err := remapTree(&nodeupConfig.ConfigStore.Keypairs, path.Join(targetDir, "pki/etcd")); err != nil { + return nil, err + } + if err := remapTree(&nodeupConfig.ConfigStore.Secrets, path.Join(targetDir, "pki")); err != nil { + return nil, err + } + } + nodeupConfigBytes, err := yaml.Marshal(nodeupConfig) if err != nil { return nil, fmt.Errorf("error converting nodeup config to yaml: %w", err) @@ -809,7 +914,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e // sum256 := sha256.Sum256(nodeupConfigBytes) // bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:]) - p := filepath.Join("/etc/kubernetes/kops/config", "igconfig", bootConfig.InstanceGroupRole.ToLowerString(), ig.Name, "nodeupconfig.yaml") + p := path.Join(targetDir, "igconfig", bootConfig.InstanceGroupRole.ToLowerString(), ig.Name, "nodeupconfig.yaml") bootstrapData.NodeupScriptAdditionalFiles[p] = nodeupConfigBytes // Copy any static manifests we need on the control plane @@ -817,7 +922,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e if !staticManifest.AppliesToRole(bootConfig.InstanceGroupRole) { continue } - p := filepath.Join("/etc/kubernetes/kops/config", staticManifest.Path) + p := path.Join(targetDir, staticManifest.Path) bootstrapData.NodeupScriptAdditionalFiles[p] = staticManifest.Contents } }