Pass AuxConfig to nodeup

cmd/kops-controller/pkg/server/node_config.go

@@ -61,6 +61,16 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
 		nodeConfig.InstanceGroupConfig = string(b)
 	}
 
+	{
+		p := s.configBase.Join("igconfig", "node", instanceGroupName, "auxconfig.yaml")
+
+		b, err := p.ReadFile()
+		if err != nil {
+			return nil, fmt.Errorf("error loading AuxConfig %q: %v", p, err)
+		}
+		nodeConfig.AuxConfig = string(b)
+	}
+
 	// We populate some certificates that we know the node will need.
 	for _, name := range []string{"ca"} {
 		cert, _, _, err := s.keystore.FindKeypair(name)

pkg/apis/nodeup/bootstrap.go

@@ -47,6 +47,9 @@ type NodeConfig struct {
 	// ClusterFullConfig holds the configuration for the cluster
 	ClusterFullConfig string `json:"clusterFullConfig,omitempty"`
 
+	// AuxConfig holds the nodeup.AuxConfig for the node's instance group.
+	AuxConfig string `json:"auxConfig,omitempty"`
+
 	// Certificates holds certificates that are already issued
 	Certificates []*NodeConfigCertificate `json:"certificates,omitempty"`
 }

pkg/client/simple/vfsclientset/clientset.go

@@ -161,6 +161,9 @@ func DeleteAllClusterState(basePath vfs.Path) error {
 		if strings.HasPrefix(relativePath, "instancegroup/") {
 			continue
 		}
+		if strings.HasPrefix(relativePath, "igconfig/") {
+			continue
+		}
 		if strings.HasPrefix(relativePath, "manifests/") {
 			continue
 		}

pkg/model/awsmodel/autoscalinggroup_test.go

@@ -68,6 +68,9 @@ func TestRootVolumeOptimizationFlag(t *testing.T) {
 				InstanceGroups:  igs,
 			},
 		},
+		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
+			Lifecycle: fi.LifecycleSync,
+		},
 		Cluster: cluster,
 	}
 
@@ -154,6 +157,9 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) {
 				InstanceGroups:  igs,
 			},
 		},
+		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
+			Lifecycle: fi.LifecycleSync,
+		},
 		Cluster: cluster,
 	}
 

pkg/model/azuremodel/BUILD.bazel

@@ -40,6 +40,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/kops:go_default_library",
+        "//pkg/model:go_default_library",
         "//pkg/model/defaults:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/azuretasks:go_default_library",

pkg/model/azuremodel/vmscaleset_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
 	"github.com/Azure/go-autorest/autorest/to"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/model/defaults"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
@@ -32,6 +33,9 @@ import (
 func TestVMScaleSetModelBuilder_Build(t *testing.T) {
 	b := VMScaleSetModelBuilder{
 		AzureModelContext: newTestAzureModelContext(),
+		BootstrapScriptBuilder: &model.BootstrapScriptBuilder{
+			Lifecycle: fi.LifecycleSync,
+		},
 	}
 	c := &fi.ModelBuilderContext{
 		Tasks: make(map[string]fi.Task),

pkg/model/bootstrapscript.go

@@ -49,6 +49,7 @@ type NodeUpConfigBuilder interface {
 
 // BootstrapScriptBuilder creates the bootstrap script
 type BootstrapScriptBuilder struct {
+	Lifecycle           fi.Lifecycle
 	NodeUpAssets        map[architectures.Architecture]*mirrors.MirroredAsset
 	NodeUpConfigBuilder NodeUpConfigBuilder
 }
@@ -66,6 +67,9 @@ type BootstrapScript struct {
 
 	// caTask holds the CA task, for dependency analysis.
 	caTask fi.Task
+
+	// auxConfig contains the nodeup auxiliary config.
+	auxConfig fi.TaskDependentResource
 }
 
 var _ fi.Task = &BootstrapScript{}
@@ -101,6 +105,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context, ca fi.R
 	}
 	sum256 := sha256.Sum256(auxData)
 	config.AuxConfigHash = base64.StdEncoding.EncodeToString(sum256[:])
+	b.auxConfig.Resource = fi.NewBytesResource(auxData)
 
 	data, err := utils.YamlMarshal(config)
 	if err != nil {
@@ -230,7 +235,15 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k
 		ca:      caTask.Certificate(),
 	}
 	task.resource.Task = task
+	task.auxConfig.Task = task
 	c.AddTask(task)
+
+	c.AddTask(&fitasks.ManagedFile{
+		Name:      fi.String("auxconfig-" + ig.Name),
+		Lifecycle: b.Lifecycle,
+		Location:  fi.String("igconfig/" + strings.ToLower(string(ig.Spec.Role)) + "/" + ig.Name + "/auxconfig.yaml"),
+		Contents:  &task.auxConfig,
+	})
 	return &task.resource, nil
 }
 

pkg/model/iam/iam_builder.go

@@ -588,6 +588,7 @@ func ReadableStatePaths(cluster *kops.Cluster, role Subject) ([]string, error) {
 			"/addons/*",
 			"/cluster.spec",
 			"/config",
+			"/igconfig/node/*",
 			"/instancegroup/*",
 			"/pki/issued/*",
 			"/pki/ssh/*",

upup/pkg/fi/cloudup/apply_cluster.go

@@ -499,6 +499,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 		return err
 	}
 	bootstrapScriptBuilder := &model.BootstrapScriptBuilder{
+		Lifecycle:           clusterLifecycle,
 		NodeUpConfigBuilder: configBuilder,
 		NodeUpAssets:        c.NodeUpAssets,
 	}

upup/pkg/fi/nodeup/command.go

@@ -18,6 +18,8 @@ package nodeup
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -64,6 +66,7 @@ type NodeUpCommand struct {
 	Target         string
 	cluster        *api.Cluster
 	config         *nodeup.Config
+	auxConfig      *nodeup.AuxConfig
 	instanceGroup  *api.InstanceGroup
 }
 
@@ -152,11 +155,18 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		}
 	}
 
+	var auxConfigHash [32]byte
 	if nodeConfig != nil {
 		c.instanceGroup = &api.InstanceGroup{}
 		if err := utils.YamlUnmarshal([]byte(nodeConfig.InstanceGroupConfig), c.instanceGroup); err != nil {
 			return fmt.Errorf("error parsing InstanceGroup config response: %v", err)
 		}
+
+		c.auxConfig = &nodeup.AuxConfig{}
+		if err := utils.YamlUnmarshal([]byte(nodeConfig.AuxConfig), c.auxConfig); err != nil {
+			return fmt.Errorf("error parsing AuxConfig config response: %v", err)
+		}
+		auxConfigHash = sha256.Sum256([]byte(nodeConfig.AuxConfig))
 	} else if c.config.InstanceGroupName != "" {
 		instanceGroupLocation := configBase.Join("instancegroup", c.config.InstanceGroupName)
 
@@ -169,8 +179,25 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		if err = utils.YamlUnmarshal(b, c.instanceGroup); err != nil {
 			return fmt.Errorf("error parsing InstanceGroup %q: %v", instanceGroupLocation, err)
 		}
+
+		auxConfigLocation := configBase.Join("igconfig", strings.ToLower(string(c.instanceGroup.Spec.Role)), c.config.InstanceGroupName, "auxconfig.yaml")
+
+		c.auxConfig = &nodeup.AuxConfig{}
+		b, err = auxConfigLocation.ReadFile()
+		if err != nil {
+			return fmt.Errorf("error loading AuxConfig %q: %v", auxConfigLocation, err)
+		}
+
+		if err = utils.YamlUnmarshal(b, c.auxConfig); err != nil {
+			return fmt.Errorf("error parsing AuxConfig %q: %v", auxConfigLocation, err)
+		}
+		auxConfigHash = sha256.Sum256(b)
 	} else {
-		klog.Warningf("No instance group defined in nodeup config")
+		return fmt.Errorf("no instance group defined in nodeup config")
+	}
+
+	if c.config.AuxConfigHash != base64.StdEncoding.EncodeToString(auxConfigHash[:]) {
+		return fmt.Errorf("auxiliary config hash mismatch")
 	}
 
 	err := evaluateSpec(c)
@@ -212,14 +239,15 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	}
 
 	modelContext := &model.NodeupModelContext{
-		Cloud:         cloud,
-		Architecture:  architecture,
-		Assets:        assetStore,
-		Cluster:       c.cluster,
-		ConfigBase:    configBase,
-		Distribution:  distribution,
-		InstanceGroup: c.instanceGroup,
-		NodeupConfig:  c.config,
+		Cloud:           cloud,
+		Architecture:    architecture,
+		Assets:          assetStore,
+		Cluster:         c.cluster,
+		ConfigBase:      configBase,
+		Distribution:    distribution,
+		InstanceGroup:   c.instanceGroup,
+		NodeupConfig:    c.config,
+		NodeupAuxConfig: c.auxConfig,
 	}
 
 	var secretStore fi.SecretStore