Mark nodes NeedsUpdate when keys they use change

nodeup/pkg/model/context.go

@@ -410,11 +410,26 @@ func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContex
 		p = filepath.Join(c.PathSrvKubernetes(), p)
 	}
 
-	certificate, privateKey, err := c.KeyStore.FindPrimaryKeypair(name)
+	// We use the keypair ID passed in nodeup.Config instead of the primary
+	// keypair so that the node will be updated when the primary keypair does
+	// not match the one that we are using.
+	keypairID := c.NodeupConfig.KeypairIDs[name]
+	if keypairID == "" {
+		// kOps bug where KeypairID was not populated for the node role.
+		return fmt.Errorf("no keypair ID for %q", name)
+	}
+
+	keyset, err := c.KeyStore.FindKeyset(name)
 	if err != nil {
 		return err
 	}
 
+	item := keyset.Items[keypairID]
+	if item == nil {
+		return fmt.Errorf("did not find keypair %s for %s", keypairID, name)
+	}
+
+	certificate := item.Certificate
 	if certificate == nil {
 		return fmt.Errorf("certificate %q not found", name)
 	}
@@ -432,6 +447,7 @@ func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContex
 		Owner:    owner,
 	})
 
+	privateKey := item.PrivateKey
 	if privateKey == nil {
 		return fmt.Errorf("private key %q not found", name)
 	}

nodeup/pkg/model/kubelet_test.go

@@ -235,7 +235,8 @@ func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {
 		Cluster:      model.Cluster,
 		Architecture: "amd64",
 		NodeupConfig: &nodeup.Config{
-			CAs: map[string]string{},
+			CAs:        map[string]string{},
+			KeypairIDs: map[string]string{},
 		},
 	}
 
@@ -248,6 +249,7 @@ func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {
 	}
 
 	nodeUpModelContext.NodeupConfig.CAs["ca"] = dummyCertificate + nextCertificate
+	nodeUpModelContext.NodeupConfig.KeypairIDs["ca"] = "3"
 
 	if err := nodeUpModelContext.Init(); err != nil {
 		return nil, err

pkg/apis/nodeup/config.go

@@ -57,6 +57,8 @@ type Config struct {
 
 	// CAs are the CA certificates to trust.
 	CAs map[string]string
+	// KeypairIDs are the IDs of keysets used to sign things.
+	KeypairIDs map[string]string
 	// DefaultMachineType is the first-listed instance machine type, used if querying instance metadata fails.
 	DefaultMachineType *string `json:",omitempty"`
 	// EnableLifecycleHook defines whether we need to complete a lifecycle hook.
@@ -122,6 +124,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 	config := Config{
 		InstanceGroupRole: role,
 		CAs:               map[string]string{},
+		KeypairIDs:        map[string]string{},
 		SysctlParameters:  instanceGroup.Spec.SysctlParameters,
 		VolumeMounts:      instanceGroup.Spec.VolumeMounts,
 	}

pkg/model/bootstrapscript.go

@@ -43,7 +43,7 @@ import (
 )
 
 type NodeUpConfigBuilder interface {
-	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caCertificates fi.Resource) (*nodeup.Config, *nodeup.AuxConfig, error)
+	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.AuxConfig, error)
 }
 
 // BootstrapScriptBuilder creates the bootstrap script
@@ -91,7 +91,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context) (string
 	}
 
 	sort.Strings(alternateNames)
-	config, auxConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTask.Certificates())
+	config, auxConfig, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, b.caTask)
 	if err != nil {
 		return "", err
 	}

pkg/model/bootstrapscript_test.go

@@ -64,7 +64,7 @@ type nodeupConfigBuilder struct {
 	cluster *kops.Cluster
 }
 
-func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, ca fi.Resource) (*nodeup.Config, *nodeup.AuxConfig, error) {
+func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.AuxConfig, error) {
 	config, auxConfig := nodeup.NewConfig(n.cluster, ig)
 	return config, auxConfig, nil
 }

pkg/model/openstackmodel/servergroup_test.go

@@ -1012,7 +1012,7 @@ func createBuilderForCluster(cluster *kops.Cluster, instanceGroups []*kops.Insta
 type nodeupConfigBuilder struct {
 }
 
-func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, ca fi.Resource) (*nodeup.Config, *nodeup.AuxConfig, error) {
+func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.AuxConfig, error) {
 	return &nodeup.Config{}, &nodeup.AuxConfig{}, nil
 }
 

upup/pkg/fi/cloudup/BUILD.bazel

@@ -69,6 +69,7 @@ go_library(
         "//upup/pkg/fi/cloudup/openstack:go_default_library",
         "//upup/pkg/fi/cloudup/terraform:go_default_library",
         "//upup/pkg/fi/cloudup/terraformWriter:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//upup/pkg/fi/loader:go_default_library",
         "//util/pkg/architectures:go_default_library",
         "//util/pkg/env:go_default_library",

upup/pkg/fi/cloudup/apply_cluster.go

@@ -70,6 +70,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/hashing"
 	"k8s.io/kops/util/pkg/mirrors"
@@ -1295,7 +1296,7 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 }
 
 // BuildConfig returns the NodeUp config and auxiliary config.
-func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caCertificates fi.Resource) (*nodeup.Config, *nodeup.AuxConfig, error) {
+func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caTask *fitasks.Keypair) (*nodeup.Config, *nodeup.AuxConfig, error) {
 	cluster := n.cluster
 
 	if ig == nil {
@@ -1322,13 +1323,17 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
 	config.InstanceGroupName = ig.ObjectMeta.Name
 	config.CloudProvider = cluster.Spec.CloudProvider
 
-	cas, err := fi.ResourceAsString(caCertificates)
+	cas, err := fi.ResourceAsString(caTask.Certificates())
 	if err != nil {
 		// CA task may not have run yet; we'll retry
 		return nil, nil, fmt.Errorf("failed to read CA certificates: %w", err)
 	}
 	config.CAs[fi.CertificateIDCA] = cas
 
+	if isMaster {
+		config.KeypairIDs[fi.CertificateIDCA] = caTask.Keyset().Primary.Id
+	}
+
 	if isMaster || useGossip {
 		for _, arch := range architectures.GetSupported() {
 			for _, a := range n.protokubeAsset[arch] {