diff --git a/cmd/kops-controller/pkg/server/node_config.go b/cmd/kops-controller/pkg/server/node_config.go index 771360acaf..305a06ecf6 100644 --- a/cmd/kops-controller/pkg/server/node_config.go +++ b/cmd/kops-controller/pkg/server/node_config.go @@ -61,6 +61,16 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest nodeConfig.InstanceGroupConfig = string(b) } + { + p := s.configBase.Join("igconfig", "node", instanceGroupName, "auxconfig.yaml") + + b, err := p.ReadFile() + if err != nil { + return nil, fmt.Errorf("error loading AuxConfig %q: %v", p, err) + } + nodeConfig.AuxConfig = string(b) + } + // We populate some certificates that we know the node will need. for _, name := range []string{"ca"} { cert, _, _, err := s.keystore.FindKeypair(name) diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go index d1cf402a1d..7013ab1cc7 100644 --- a/pkg/apis/nodeup/bootstrap.go +++ b/pkg/apis/nodeup/bootstrap.go @@ -47,6 +47,9 @@ type NodeConfig struct { // ClusterFullConfig holds the configuration for the cluster ClusterFullConfig string `json:"clusterFullConfig,omitempty"` + // AuxConfig holds the nodeup.AuxConfig for the node's instance group. + AuxConfig string `json:"auxConfig,omitempty"` + // Certificates holds certificates that are already issued Certificates []*NodeConfigCertificate `json:"certificates,omitempty"` } diff --git a/pkg/client/simple/vfsclientset/clientset.go b/pkg/client/simple/vfsclientset/clientset.go index e15a94aa6a..46b72b381b 100644 --- a/pkg/client/simple/vfsclientset/clientset.go +++ b/pkg/client/simple/vfsclientset/clientset.go @@ -161,6 +161,9 @@ func DeleteAllClusterState(basePath vfs.Path) error { if strings.HasPrefix(relativePath, "instancegroup/") { continue } + if strings.HasPrefix(relativePath, "igconfig/") { + continue + } if strings.HasPrefix(relativePath, "manifests/") { continue } diff --git a/pkg/model/awsmodel/autoscalinggroup_test.go b/pkg/model/awsmodel/autoscalinggroup_test.go index 32369b1b19..6070996ce1 100644 --- a/pkg/model/awsmodel/autoscalinggroup_test.go +++ b/pkg/model/awsmodel/autoscalinggroup_test.go @@ -68,6 +68,9 @@ func TestRootVolumeOptimizationFlag(t *testing.T) { InstanceGroups: igs, }, }, + BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ + Lifecycle: fi.LifecycleSync, + }, Cluster: cluster, } @@ -154,6 +157,9 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) { InstanceGroups: igs, }, }, + BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ + Lifecycle: fi.LifecycleSync, + }, Cluster: cluster, } diff --git a/pkg/model/azuremodel/BUILD.bazel b/pkg/model/azuremodel/BUILD.bazel index 54311a2a73..24b8de5e27 100644 --- a/pkg/model/azuremodel/BUILD.bazel +++ b/pkg/model/azuremodel/BUILD.bazel @@ -40,6 +40,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/kops:go_default_library", + "//pkg/model:go_default_library", "//pkg/model/defaults:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/azuretasks:go_default_library", diff --git a/pkg/model/azuremodel/vmscaleset_test.go b/pkg/model/azuremodel/vmscaleset_test.go index 986287a0f2..38f8cf86a1 100644 --- a/pkg/model/azuremodel/vmscaleset_test.go +++ b/pkg/model/azuremodel/vmscaleset_test.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" "github.com/Azure/go-autorest/autorest/to" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/model/defaults" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" @@ -32,6 +33,9 @@ import ( func TestVMScaleSetModelBuilder_Build(t *testing.T) { b := VMScaleSetModelBuilder{ AzureModelContext: newTestAzureModelContext(), + BootstrapScriptBuilder: &model.BootstrapScriptBuilder{ + Lifecycle: fi.LifecycleSync, + }, } c := &fi.ModelBuilderContext{ Tasks: make(map[string]fi.Task), diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 39c66bd28f..c03880f5a5 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -49,6 +49,7 @@ type NodeUpConfigBuilder interface { // BootstrapScriptBuilder creates the bootstrap script type BootstrapScriptBuilder struct { + Lifecycle fi.Lifecycle NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset NodeUpConfigBuilder NodeUpConfigBuilder } @@ -66,6 +67,9 @@ type BootstrapScript struct { // caTask holds the CA task, for dependency analysis. caTask fi.Task + + // auxConfig contains the nodeup auxiliary config. + auxConfig fi.TaskDependentResource } var _ fi.Task = &BootstrapScript{} @@ -101,6 +105,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context, ca fi.R } sum256 := sha256.Sum256(auxData) config.AuxConfigHash = base64.StdEncoding.EncodeToString(sum256[:]) + b.auxConfig.Resource = fi.NewBytesResource(auxData) data, err := utils.YamlMarshal(config) if err != nil { @@ -230,7 +235,15 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k ca: caTask.Certificate(), } task.resource.Task = task + task.auxConfig.Task = task c.AddTask(task) + + c.AddTask(&fitasks.ManagedFile{ + Name: fi.String("auxconfig-" + ig.Name), + Lifecycle: b.Lifecycle, + Location: fi.String("igconfig/" + strings.ToLower(string(ig.Spec.Role)) + "/" + ig.Name + "/auxconfig.yaml"), + Contents: &task.auxConfig, + }) return &task.resource, nil } diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index e15c4512c7..5d6a748ef1 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -588,6 +588,7 @@ func ReadableStatePaths(cluster *kops.Cluster, role Subject) ([]string, error) { "/addons/*", "/cluster.spec", "/config", + "/igconfig/node/*", "/instancegroup/*", "/pki/issued/*", "/pki/ssh/*", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 410501838c..993e27119b 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -499,6 +499,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { return err } bootstrapScriptBuilder := &model.BootstrapScriptBuilder{ + Lifecycle: clusterLifecycle, NodeUpConfigBuilder: configBuilder, NodeUpAssets: c.NodeUpAssets, } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 33ab90a7c6..1a8ec7b29b 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -18,6 +18,8 @@ package nodeup import ( "context" + "crypto/sha256" + "encoding/base64" "errors" "fmt" "io" @@ -64,6 +66,7 @@ type NodeUpCommand struct { Target string cluster *api.Cluster config *nodeup.Config + auxConfig *nodeup.AuxConfig instanceGroup *api.InstanceGroup } @@ -152,11 +155,18 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } } + var auxConfigHash [32]byte if nodeConfig != nil { c.instanceGroup = &api.InstanceGroup{} if err := utils.YamlUnmarshal([]byte(nodeConfig.InstanceGroupConfig), c.instanceGroup); err != nil { return fmt.Errorf("error parsing InstanceGroup config response: %v", err) } + + c.auxConfig = &nodeup.AuxConfig{} + if err := utils.YamlUnmarshal([]byte(nodeConfig.AuxConfig), c.auxConfig); err != nil { + return fmt.Errorf("error parsing AuxConfig config response: %v", err) + } + auxConfigHash = sha256.Sum256([]byte(nodeConfig.AuxConfig)) } else if c.config.InstanceGroupName != "" { instanceGroupLocation := configBase.Join("instancegroup", c.config.InstanceGroupName) @@ -169,8 +179,25 @@ func (c *NodeUpCommand) Run(out io.Writer) error { if err = utils.YamlUnmarshal(b, c.instanceGroup); err != nil { return fmt.Errorf("error parsing InstanceGroup %q: %v", instanceGroupLocation, err) } + + auxConfigLocation := configBase.Join("igconfig", strings.ToLower(string(c.instanceGroup.Spec.Role)), c.config.InstanceGroupName, "auxconfig.yaml") + + c.auxConfig = &nodeup.AuxConfig{} + b, err = auxConfigLocation.ReadFile() + if err != nil { + return fmt.Errorf("error loading AuxConfig %q: %v", auxConfigLocation, err) + } + + if err = utils.YamlUnmarshal(b, c.auxConfig); err != nil { + return fmt.Errorf("error parsing AuxConfig %q: %v", auxConfigLocation, err) + } + auxConfigHash = sha256.Sum256(b) } else { - klog.Warningf("No instance group defined in nodeup config") + return fmt.Errorf("no instance group defined in nodeup config") + } + + if c.config.AuxConfigHash != base64.StdEncoding.EncodeToString(auxConfigHash[:]) { + return fmt.Errorf("auxiliary config hash mismatch") } err := evaluateSpec(c) @@ -212,14 +239,15 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } modelContext := &model.NodeupModelContext{ - Cloud: cloud, - Architecture: architecture, - Assets: assetStore, - Cluster: c.cluster, - ConfigBase: configBase, - Distribution: distribution, - InstanceGroup: c.instanceGroup, - NodeupConfig: c.config, + Cloud: cloud, + Architecture: architecture, + Assets: assetStore, + Cluster: c.cluster, + ConfigBase: configBase, + Distribution: distribution, + InstanceGroup: c.instanceGroup, + NodeupConfig: c.config, + NodeupAuxConfig: c.auxConfig, } var secretStore fi.SecretStore