Merge pull request #10469 from justinsb/boot_nodes_from_kops_controller

Boot nodes without state store access

cmd/kops-controller/pkg/server/BUILD.bazel

@@ -4,16 +4,19 @@ go_library(
     name = "go_default_library",
     srcs = [
         "keystore.go",
+        "node_config.go",
         "server.go",
     ],
     importpath = "k8s.io/kops/cmd/kops-controller/pkg/server",
     visibility = ["//visibility:public"],
     deps = [
         "//cmd/kops-controller/pkg/config:go_default_library",
+        "//pkg/apis/kops/registry:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/pki:go_default_library",
         "//pkg/rbac:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//util/pkg/vfs:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],

cmd/kops-controller/pkg/server/node_config.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops/registry"
+	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest, identity *fi.VerifyResult) (*nodeup.NodeConfig, error) {
+	klog.Infof("getting node config for %+v", req)
+
+	instanceGroupName := identity.InstanceGroupName
+	if instanceGroupName == "" {
+		return nil, fmt.Errorf("did not find InstanceGroup for node %q", identity.NodeName)
+	}
+
+	nodeConfig := &nodeup.NodeConfig{}
+
+	// Note: For now, we're assuming there is only a single cluster, and it is ours.
+	// We therefore use the configured base path
+
+	// Today we load the full cluster config from the state store (e.g. S3) every time
+	// TODO: we should generate it on the fly (to allow for cluster reconfiguration)
+	{
+		p := s.configBase.Join(registry.PathClusterCompleted)
+
+		b, err := p.ReadFile()
+		if err != nil {
+			return nil, fmt.Errorf("error loading cluster config %q: %w", p, err)
+		}
+		nodeConfig.ClusterFullConfig = string(b)
+	}
+
+	{
+		p := s.configBase.Join("instancegroup", instanceGroupName)
+
+		b, err := p.ReadFile()
+		if err != nil {
+			return nil, fmt.Errorf("error loading InstanceGroup %q: %v", p, err)
+		}
+		nodeConfig.InstanceGroupConfig = string(b)
+	}
+
+	// We populate some certificates that we know the node will need.
+	for _, name := range []string{"ca"} {
+		cert, _, _, err := s.keystore.FindKeypair(name)
+		if err != nil {
+			return nil, fmt.Errorf("error getting certificate %q: %w", name, err)
+		}
+
+		if cert == nil {
+			return nil, fmt.Errorf("certificate %q not found", name)
+		}
+
+		certData, err := cert.AsString()
+		if err != nil {
+			return nil, fmt.Errorf("error marshalling certificate %q: %w", name, err)
+		}
+
+		nodeConfig.Certificates = append(nodeConfig.Certificates, &nodeup.NodeConfigCertificate{
+			Name: name,
+			Cert: certData,
+		})
+	}
+
+	return nodeConfig, nil
+}

cmd/kops-controller/pkg/server/server.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/pkg/rbac"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 type Server struct {
@@ -44,6 +45,9 @@ type Server struct {
 	server    *http.Server
 	verifier  fi.Verifier
 	keystore  pki.Keystore
+
+	// configBase is the base of the configuration storage.
+	configBase vfs.Path
 }
 
 func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) {
@@ -61,6 +65,13 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) {
 		server:    server,
 		verifier:  verifier,
 	}
+
+	configBase, err := vfs.Context.BuildVfsPath(opt.ConfigBase)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse ConfigBase %q: %v", opt.ConfigBase, err)
+	}
+	s.configBase = configBase
+
 	r := http.NewServeMux()
 	r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap))
 	server.Handler = recovery(r)
@@ -121,6 +132,18 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 		Certs: map[string]string{},
 	}
 
+	// Support for nodes that have no access to the state store
+	if req.IncludeNodeConfig {
+		nodeConfig, err := s.getNodeConfig(r.Context(), req, id)
+		if err != nil {
+			klog.Infof("bootstrap failed to build node config: %v", err)
+			w.WriteHeader(http.StatusBadRequest)
+			_, _ = w.Write([]byte("failed to build node config"))
+			return
+		}
+		resp.NodeConfig = nodeConfig
+	}
+
 	// Skew the certificate lifetime by up to 30 days based on information about the requesting node.
 	// This is so that different nodes created at the same time have the certificates they generated
 	// expire at different times, but all certificates on a given node expire around the same time.

nodeup/pkg/model/bootstrap_client.go

@@ -18,8 +18,12 @@ package model
 
 import (
 	"fmt"
+	"net"
+	"net/url"
+	"strconv"
 
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -56,18 +60,29 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	bootstrapClient := &nodetasks.BootstrapClient{
+	baseURL := url.URL{
+		Scheme: "https",
+		Host:   net.JoinHostPort("kops-controller.internal."+b.Cluster.ObjectMeta.Name, strconv.Itoa(wellknownports.KopsControllerPort)),
+		Path:   "/",
+	}
+
+	bootstrapClient := &nodetasks.KopsBootstrapClient{
 		Authenticator: authenticator,
 		CA:            cert,
-		Certs:         b.bootstrapCerts,
+		BaseURL:       baseURL,
+	}
+
+	bootstrapClientTask := &nodetasks.BootstrapClientTask{
+		Client: bootstrapClient,
+		Certs:  b.bootstrapCerts,
 	}
 
 	for _, cert := range b.bootstrapCerts {
-		cert.Cert.Task = bootstrapClient
-		cert.Key.Task = bootstrapClient
+		cert.Cert.Task = bootstrapClientTask
+		cert.Key.Task = bootstrapClientTask
 	}
 
-	c.AddTask(bootstrapClient)
+	c.AddTask(bootstrapClientTask)
 	return nil
 }
 

pkg/apis/nodeup/bootstrap.go

@@ -24,10 +24,38 @@ type BootstrapRequest struct {
 	APIVersion string `json:"apiVersion"`
 	// Certs are the requested certificates and their respective public keys.
 	Certs map[string]string `json:"certs"`
+
+	// IncludeNodeConfig controls whether the cluster & instance group configuration should be returned.
+	// This allows for nodes without access to the kops state store.
+	IncludeNodeConfig bool `json:"includeNodeConfig"`
 }
 
 // BootstrapResponse is a response to a BootstrapRequest.
 type BootstrapResponse struct {
 	// Certs are the issued certificates.
 	Certs map[string]string
+
+	// NodeConfig contains the node configuration, if IncludeNodeConfig is set.
+	NodeConfig *NodeConfig `json:"nodeConfig,omitempty"`
+}
+
+// NodeConfig holds configuration needed to boot a node (without the kops state store)
+type NodeConfig struct {
+	// InstanceGroupConfig holds the configuration for the node's instance group
+	InstanceGroupConfig string `json:"instanceGroupConfig,omitempty"`
+
+	// ClusterFullConfig holds the configuration for the cluster
+	ClusterFullConfig string `json:"clusterFullConfig,omitempty"`
+
+	// Certificates holds certificates that are already issued
+	Certificates []*NodeConfigCertificate `json:"certificates,omitempty"`
+}
+
+// NodeConfigCertificate holds a certificate that the node needs to boot.
+type NodeConfigCertificate struct {
+	// Name identifies the certificate.
+	Name string `json:"name,omitempty"`
+
+	// Cert is the certificate data.
+	Cert string `json:"cert,omitempty"`
 }

pkg/apis/nodeup/config.go

@@ -66,6 +66,19 @@ type Config struct {
 	SysctlParameters []string `json:",omitempty"`
 	// VolumeMounts are a collection of volume mounts.
 	VolumeMounts []kops.VolumeMountSpec `json:",omitempty"`
+
+	// ConfigServer holds the configuration for the configuration server
+	ConfigServer *ConfigServerOptions `json:"configServer,omitempty"`
+}
+
+type ConfigServerOptions struct {
+	// Server is the address of the configuration server to use (kops-controller)
+	Server string `json:"server,omitempty"`
+	// CA is the ca-certificate to require for the configuration server
+	CA string `json:"ca,omitempty"`
+
+	// CloudProvider is the cloud provider in use (needed for authentication)
+	CloudProvider string `json:"cloudProvider,omitempty"`
 }
 
 // Image is a docker image we should pre-load

pkg/configserver/BUILD.bazel

@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "keystore.go",
+        "secretstore.go",
+    ],
+    importpath = "k8s.io/kops/pkg/configserver",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//pkg/apis/nodeup:go_default_library",
+        "//pkg/pki:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//util/pkg/vfs:go_default_library",
+    ],
+)

pkg/configserver/keystore.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configserver
+
+import (
+	"crypto/x509"
+	"fmt"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+//configserverKeyStore is a KeyStore backed by the config server.
+type configserverKeyStore struct {
+	nodeConfig *nodeup.NodeConfig
+}
+
+func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore {
+	return &configserverKeyStore{
+		nodeConfig: nodeConfig,
+	}
+}
+
+// FindKeypair implements fi.Keystore
+func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+	return nil, nil, false, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name)
+}
+
+// FindKeypair implements fi.Keystore
+func (s *configserverKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
+	return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore")
+}
+
+// FindKeypair implements fi.Keystore
+func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+	return fmt.Errorf("StoreKeypair not supported by configserverKeyStore")
+}
+
+// FindKeypair implements fi.Keystore
+func (s *configserverKeyStore) MirrorTo(basedir vfs.Path) error {
+	return fmt.Errorf("MirrorTo not supported by configserverKeyStore")
+}
+
+// CertificatePool implements fi.CAStore
+func (s *configserverKeyStore) CertificatePool(name string, createIfMissing bool) (*fi.CertificatePool, error) {
+	return nil, fmt.Errorf("CertificatePool not supported by configserverKeyStore")
+}
+
+// FindCertificatePool implements fi.CAStore
+func (s *configserverKeyStore) FindCertificatePool(name string) (*fi.CertificatePool, error) {
+	return nil, fmt.Errorf("FindCertificatePool not supported by configserverKeyStore")
+}
+
+// FindCertificateKeyset implements fi.CAStore
+func (s *configserverKeyStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
+	return nil, fmt.Errorf("FindCertificateKeyset not supported by configserverKeyStore")
+}
+
+// FindPrivateKey implements fi.CAStore
+func (s *configserverKeyStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
+	return nil, fmt.Errorf("FindPrivateKey not supported by configserverKeyStore")
+}
+
+// FindPrivateKeyset implements fi.CAStore
+func (s *configserverKeyStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
+	return nil, fmt.Errorf("FindPrivateKeyset not supported by configserverKeyStore")
+}
+
+// FindCert implements fi.CAStore
+func (s *configserverKeyStore) FindCert(name string) (*pki.Certificate, error) {
+	for _, cert := range s.nodeConfig.Certificates {
+		if cert.Name == name {
+			// Special case for the CA certificate
+			c, err := pki.ParsePEMCertificate([]byte(cert.Cert))
+			if err != nil {
+				return nil, fmt.Errorf("error parsing certificate %q: %w", name, err)
+			}
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("FindCert(%q) not supported by configserverKeyStore", name)
+}
+
+// ListKeysets implements fi.CAStore
+func (s *configserverKeyStore) ListKeysets() ([]*kops.Keyset, error) {
+	return nil, fmt.Errorf("ListKeysets not supported by configserverKeyStore")
+}
+
+// AddCert implements fi.CAStore
+func (s *configserverKeyStore) AddCert(name string, cert *pki.Certificate) error {
+	return fmt.Errorf("AddCert not supported by configserverKeyStore")
+}
+
+// DeleteKeysetItem implements fi.CAStore
+func (s *configserverKeyStore) DeleteKeysetItem(item *kops.Keyset, id string) error {
+	return fmt.Errorf("DeleteKeysetItem not supported by configserverKeyStore")
+}

pkg/configserver/secretstore.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configserver
+
+import (
+	"fmt"
+
+	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+//configserverSecretStore is a SecretStore backed by the config server.
+type configserverSecretStore struct {
+	nodeConfig *nodeup.NodeConfig
+}
+
+func NewSecretStore(nodeConfig *nodeup.NodeConfig) fi.SecretStore {
+	return &configserverSecretStore{
+		nodeConfig: nodeConfig,
+	}
+}
+
+// Secret implements fi.SecretStore
+func (s *configserverSecretStore) Secret(id string) (*fi.Secret, error) {
+	return nil, fmt.Errorf("Secret not supported by configserverSecretStore")
+}
+
+// DeleteSecret implements fi.SecretStore
+func (s *configserverSecretStore) DeleteSecret(id string) error {
+	return fmt.Errorf("DeleteSecret not supported by configserverSecretStore")
+}
+
+// FindSecret implements fi.SecretStore
+func (s *configserverSecretStore) FindSecret(id string) (*fi.Secret, error) {
+	return nil, fmt.Errorf("FindSecret not supported by configserverSecretStore")
+}
+
+// GetOrCreateSecret implements fi.SecretStore
+func (s *configserverSecretStore) GetOrCreateSecret(id string, secret *fi.Secret) (current *fi.Secret, created bool, err error) {
+	return nil, false, fmt.Errorf("GetOrCreateSecret not supported by configserverSecretStore")
+}
+
+// ReplaceSecret implements fi.SecretStore
+func (s *configserverSecretStore) ReplaceSecret(id string, secret *fi.Secret) (current *fi.Secret, err error) {
+	return nil, fmt.Errorf("ReplaceSecret not supported by configserverSecretStore")
+}
+
+// ListSecrets implements fi.SecretStore
+func (s *configserverSecretStore) ListSecrets() ([]string, error) {
+	return nil, fmt.Errorf("ListSecrets not supported by configserverSecretStore")
+}
+
+// MirrorTo implements fi.SecretStore
+func (s *configserverSecretStore) MirrorTo(basedir vfs.Path) error {
+	return fmt.Errorf("MirrorTo not supported by configserverSecretStore")
+}

pkg/featureflag/featureflag.go

@@ -99,6 +99,8 @@ var (
 	PublicJWKS = New("PublicJWKS", Bool(false))
 	// Azure toggles the Azure support.
 	Azure = New("Azure", Bool(false))
+	// KopsControllerStateStore enables fetching the kops state from kops-controller, instead of requiring access to S3/GCS/etc.
+	KopsControllerStateStore = New("KopsControllerStateStore", Bool(false))
 )
 
 // FeatureFlag defines a feature flag

pkg/model/BUILD.bazel

@@ -80,6 +80,7 @@ go_test(
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/testutils/golden:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//util/pkg/architectures:go_default_library",
         "//util/pkg/hashing:go_default_library",
         "//util/pkg/mirrors:go_default_library",

pkg/model/awsmodel/BUILD.bazel

@@ -39,6 +39,7 @@ go_test(
         "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awstasks:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )

pkg/model/awsmodel/autoscalinggroup_test.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kops/pkg/testutils"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 )
 
 const (
@@ -73,7 +74,17 @@ func TestRootVolumeOptimizationFlag(t *testing.T) {
 		Tasks: make(map[string]fi.Task),
 	}
 
-	b.Build(c)
+	// We need the CA for the bootstrap script
+	caTask := &fitasks.Keypair{
+		Name:    fi.String(fi.CertificateIDCA),
+		Subject: "cn=kubernetes",
+		Type:    "ca",
+	}
+	c.AddTask(caTask)
+
+	if err := b.Build(c); err != nil {
+		t.Fatalf("error from Build: %v", err)
+	}
 
 	lc := c.Tasks["LaunchTemplate/nodes.testcluster.test.com"].(*awstasks.LaunchTemplate)
 
@@ -148,6 +159,14 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) {
 		Tasks: make(map[string]fi.Task),
 	}
 
+	// We need the CA for the bootstrap script
+	caTask := &fitasks.Keypair{
+		Name:    fi.String(fi.CertificateIDCA),
+		Subject: "cn=kubernetes",
+		Type:    "ca",
+	}
+	c.AddTask(caTask)
+
 	b.Build(c)
 
 	hasSecurityGroup := func(lt *awstasks.LaunchTemplate, id string) bool {

pkg/model/azuremodel/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
         "//pkg/model/defaults:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/azuretasks:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
     ],

pkg/model/azuremodel/vmscaleset_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/model/defaults"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 )
 
 func TestVMScaleSetModelBuilder_Build(t *testing.T) {
@@ -35,6 +36,14 @@ func TestVMScaleSetModelBuilder_Build(t *testing.T) {
 	c := &fi.ModelBuilderContext{
 		Tasks: make(map[string]fi.Task),
 	}
+
+	caTask := &fitasks.Keypair{
+		Name:    fi.String(fi.CertificateIDCA),
+		Subject: "cn=kubernetes",
+		Type:    "ca",
+	}
+	c.AddTask(caTask)
+
 	err := b.Build(c)
 	if err != nil {
 		t.Errorf("unexpected error %s", err)

pkg/model/bootstrapscript.go

@@ -37,12 +37,13 @@ import (
 	"k8s.io/kops/pkg/model/resources"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/mirrors"
 )
 
 type NodeUpConfigBuilder interface {
-	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string) (*nodeup.Config, error)
+	BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, ca fi.Resource) (*nodeup.Config, error)
 }
 
 // BootstrapScriptBuilder creates the bootstrap script
@@ -58,6 +59,12 @@ type BootstrapScript struct {
 	resource fi.TaskDependentResource
 	// alternateNameTasks are tasks that contribute api-server IP addresses.
 	alternateNameTasks []fi.HasAddress
+
+	// ca holds the CA certificate
+	ca fi.Resource
+
+	// caTask holds the CA task, for dependency analysis.
+	caTask fi.Task
 }
 
 var _ fi.Task = &BootstrapScript{}
@@ -65,7 +72,7 @@ var _ fi.HasName = &BootstrapScript{}
 var _ fi.HasDependencies = &BootstrapScript{}
 
 // kubeEnv returns the nodeup config for the instance group
-func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context) (string, error) {
+func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context, ca fi.Resource) (string, error) {
 	var alternateNames []string
 
 	for _, hasAddress := range b.alternateNameTasks {
@@ -82,7 +89,7 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.Context) (string
 	}
 
 	sort.Strings(alternateNames)
-	config, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames)
+	config, err := b.builder.NodeUpConfigBuilder.BuildConfig(ig, alternateNames, ca)
 	if err != nil {
 		return "", err
 	}
@@ -192,6 +199,12 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[
 // ResourceNodeUp generates and returns a nodeup (bootstrap) script from a
 // template file, substituting in specific env vars & cluster spec configuration
 func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *kops.InstanceGroup) (fi.Resource, error) {
+	caTaskObject, found := c.Tasks["Keypair/ca"]
+	if !found {
+		return nil, fmt.Errorf("keypair/ca task not found")
+	}
+	caTask := caTaskObject.(*fitasks.Keypair)
+
 	// Bastions can have AdditionalUserData, but if there isn't any skip this part
 	if ig.IsBastion() && len(ig.Spec.AdditionalUserData) == 0 {
 		templateResource, err := NewTemplateResource("nodeup", "", nil, nil)
@@ -205,6 +218,8 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k
 		Name:    ig.Name,
 		ig:      ig,
 		builder: b,
+		caTask:  caTask,
+		ca:      caTask.Certificate(),
 	}
 	task.resource.Task = task
 	c.AddTask(task)
@@ -225,6 +240,8 @@ func (b *BootstrapScript) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 		}
 	}
 
+	deps = append(deps, b.caTask)
+
 	return deps
 }
 
@@ -255,7 +272,7 @@ func (b *BootstrapScript) Run(c *fi.Context) error {
 			return ""
 		},
 		"KubeEnv": func() (string, error) {
-			return b.kubeEnv(b.ig, c)
+			return b.kubeEnv(b.ig, c, b.ca)
 		},
 
 		"EnvironmentVariables": func() (string, error) {

pkg/model/bootstrapscript_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/testutils/golden"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/hashing"
 	"k8s.io/kops/util/pkg/mirrors"
@@ -62,7 +63,7 @@ type nodeupConfigBuilder struct {
 	cluster *kops.Cluster
 }
 
-func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string) (*nodeup.Config, error) {
+func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, ca fi.Resource) (*nodeup.Config, error) {
 	return nodeup.NewConfig(n.cluster, ig), nil
 }
 
@@ -130,6 +131,13 @@ func TestBootstrapUserData(t *testing.T) {
 			Tasks: make(map[string]fi.Task),
 		}
 
+		caTask := &fitasks.Keypair{
+			Name:    fi.String(fi.CertificateIDCA),
+			Subject: "cn=kubernetes",
+			Type:    "ca",
+		}
+		c.AddTask(caTask)
+
 		bs := &BootstrapScriptBuilder{
 			NodeUpConfigBuilder: &nodeupConfigBuilder{cluster: cluster},
 			NodeUpAssets: map[architectures.Architecture]*mirrors.MirroredAsset{

pkg/model/openstackmodel/BUILD.bazel

@@ -39,6 +39,7 @@ go_test(
         "//pkg/model/iam:go_default_library",
         "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//util/pkg/architectures:go_default_library",
         "//util/pkg/hashing:go_default_library",
         "//util/pkg/mirrors:go_default_library",

pkg/model/openstackmodel/servergroup_test.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/pkg/model/iam"
 	"k8s.io/kops/pkg/testutils"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/hashing"
 	"k8s.io/kops/util/pkg/mirrors"
@@ -972,7 +973,7 @@ func createBuilderForCluster(cluster *kops.Cluster, instanceGroups []*kops.Insta
 type nodeupConfigBuilder struct {
 }
 
-func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string) (*nodeup.Config, error) {
+func (n *nodeupConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, ca fi.Resource) (*nodeup.Config, error) {
 	return &nodeup.Config{}, nil
 }
 
@@ -1012,7 +1013,17 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde
 		LifecycleOverrides: map[string]fi.Lifecycle{},
 	}
 
-	builder.Build(context)
+	// We need the CA for the bootstrap script
+	caTask := &fitasks.Keypair{
+		Name:    fi.String(fi.CertificateIDCA),
+		Subject: "cn=kubernetes",
+		Type:    "ca",
+	}
+	context.AddTask(caTask)
+
+	if err := builder.Build(context); err != nil {
+		t.Fatalf("error from Build: %v", err)
+	}
 
 	file := filepath.Join(basedir, strings.ReplaceAll(testCase.desc, " ", "-")+".yaml")
 

pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml

@@ -68,6 +68,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups:
 - additional-sg
 ID: null

pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml

@@ -67,6 +67,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml

@@ -67,6 +67,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml

@@ -497,6 +497,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml

@@ -468,6 +468,14 @@ UserData:
   task:
     Name: node-c
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 ID: null
 Lifecycle: Sync
 Name: master-public-name

pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml

@@ -505,6 +505,14 @@ UserData:
   task:
     Name: node-c
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml

@@ -427,6 +427,14 @@ UserData:
   task:
     Name: node-c
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml

@@ -208,6 +208,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml

@@ -234,6 +234,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml

@@ -141,6 +141,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml

@@ -167,6 +167,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups: null
 ID: null
 Lifecycle: Sync

pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml

@@ -68,6 +68,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups:
 - additional-sg
 ID: null

pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml

@@ -68,6 +68,14 @@ UserData:
   task:
     Name: node
 ---
+Lifecycle: null
+Name: ca
+Signer: null
+alternateNames: null
+oldFormat: false
+subject: cn=kubernetes
+type: ca
+---
 AdditionalSecurityGroups:
 - additional-sg
 ID: null

upup/pkg/fi/authenticate.go

@@ -25,6 +25,9 @@ type Authenticator interface {
 type VerifyResult struct {
 	// Nodename is the name that this node is authorized to use.
 	NodeName string
+
+	// InstanceGroupName is the name of the kops InstanceGroup this node is a member of.
+	InstanceGroupName string
 }
 
 // Verifier verifies authentication credentials for requests.

upup/pkg/fi/cloudup/apply_cluster.go

@@ -20,9 +20,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/blang/semver/v4"
@@ -54,6 +56,7 @@ import (
 	"k8s.io/kops/pkg/model/spotinstmodel"
 	"k8s.io/kops/pkg/resources/digitalocean"
 	"k8s.io/kops/pkg/templates"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/models"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/aliup"
@@ -1328,11 +1331,12 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 		images:         images,
 		protokubeImage: protokubeImage,
 	}
+
 	return &configBuilder, nil
 }
 
 // BuildNodeUpConfig returns the NodeUp config, in YAML format
-func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string) (*nodeup.Config, error) {
+func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAdditionalIPs []string, caResource fi.Resource) (*nodeup.Config, error) {
 	cluster := n.cluster
 
 	if ig == nil {
@@ -1353,9 +1357,33 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
 		}
 	}
 	config.ClusterName = cluster.ObjectMeta.Name
-	config.ConfigBase = fi.String(n.configBase.Path())
 	config.InstanceGroupName = ig.ObjectMeta.Name
 
+	useConfigServer := featureflag.KopsControllerStateStore.Enabled() && (role != kops.InstanceGroupRoleMaster)
+	if useConfigServer {
+		baseURL := url.URL{
+			Scheme: "https",
+			Host:   net.JoinHostPort("kops-controller.internal."+cluster.ObjectMeta.Name, strconv.Itoa(wellknownports.KopsControllerPort)),
+			Path:   "/",
+		}
+
+		ca, err := fi.ResourceAsString(caResource)
+		if err != nil {
+			// CA task may not have run yet; we'll retry
+			return nil, fmt.Errorf("failed to read CA certificate: %w", err)
+		}
+
+		configServer := &nodeup.ConfigServerOptions{
+			Server:        baseURL.String(),
+			CloudProvider: cluster.Spec.CloudProvider,
+			CA:            ca,
+		}
+
+		config.ConfigServer = configServer
+	} else {
+		config.ConfigBase = fi.String(n.configBase.Path())
+	}
+
 	if role == kops.InstanceGroupRoleMaster {
 		config.ApiserverAdditionalIPs = apiserverAdditionalIPs
 	}

upup/pkg/fi/cloudup/awsup/BUILD.bazel

@@ -32,6 +32,7 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/aws/arn:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/endpoints:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",

upup/pkg/fi/cloudup/awsup/aws_authenticator.go

@@ -17,11 +17,14 @@ limitations under the License.
 package awsup
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"k8s.io/kops/upup/pkg/fi"
@@ -35,6 +38,24 @@ type awsAuthenticator struct {
 
 var _ fi.Authenticator = &awsAuthenticator{}
 
+// RegionFromMetadata returns the current region from the aws metdata
+func RegionFromMetadata(ctx context.Context) (string, error) {
+	config := aws.NewConfig()
+	config = config.WithCredentialsChainVerboseErrors(true)
+
+	s, err := session.NewSession(config)
+	if err != nil {
+		return "", err
+	}
+	metadata := ec2metadata.New(s, config)
+
+	region, err := metadata.RegionWithContext(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to get region from ec2 metadata: %w", err)
+	}
+	return region, nil
+}
+
 func NewAWSAuthenticator(region string) (fi.Authenticator, error) {
 	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true).WithRegion(region)
 	sess, err := session.NewSession(config)

upup/pkg/fi/cloudup/awsup/aws_verifier.go

@@ -35,6 +35,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/sts"
+	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -166,17 +167,17 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e
 		return nil, fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody))
 	}
 
-	result := GetCallerIdentityResponse{}
-	err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&result)
+	callerIdentity := GetCallerIdentityResponse{}
+	err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&callerIdentity)
 	if err != nil {
 		return nil, fmt.Errorf("decoding STS response: %v", err)
 	}
 
-	if result.GetCallerIdentityResult[0].Account != a.accountId {
-		return nil, fmt.Errorf("incorrect account %s", result.GetCallerIdentityResult[0].Account)
+	if callerIdentity.GetCallerIdentityResult[0].Account != a.accountId {
+		return nil, fmt.Errorf("incorrect account %s", callerIdentity.GetCallerIdentityResult[0].Account)
 	}
 
-	arn := result.GetCallerIdentityResult[0].Arn
+	arn := callerIdentity.GetCallerIdentityResult[0].Arn
 	parts := strings.Split(arn, ":")
 	if len(parts) != 6 {
 		return nil, fmt.Errorf("arn %q contains unexpected number of colons", arn)
@@ -225,7 +226,18 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e
 		return nil, fmt.Errorf("found multiple instances with instance id: %s", instanceID)
 	}
 
-	return &fi.VerifyResult{
-		NodeName: aws.StringValue(instances.Reservations[0].Instances[0].PrivateDnsName),
-	}, nil
+	instance := instances.Reservations[0].Instances[0]
+
+	result := &fi.VerifyResult{
+		NodeName: aws.StringValue(instance.PrivateDnsName),
+	}
+
+	for _, tag := range instance.Tags {
+		tagKey := aws.StringValue(tag.Key)
+		if tagKey == nodeidentityaws.CloudTagInstanceGroupName {
+			result.InstanceGroupName = aws.StringValue(tag.Value)
+		}
+	}
+
+	return result, nil
 }

upup/pkg/fi/fitasks/keypair.go

@@ -310,3 +310,8 @@ func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource {
 	e.ensureResources()
 	return e.certificateSHA1Fingerprint
 }
+
+func (e *Keypair) Certificate() fi.Resource {
+	e.ensureResources()
+	return e.certificate
+}

upup/pkg/fi/nodeup/BUILD.bazel

@@ -15,7 +15,9 @@ go_library(
         "//pkg/apis/kops/registry:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/assets:go_default_library",
+        "//pkg/configserver:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/nodeup/cloudinit:go_default_library",
         "//upup/pkg/fi/nodeup/local:go_default_library",
         "//upup/pkg/fi/nodeup/nodetasks:go_default_library",

upup/pkg/fi/nodeup/command.go

@@ -17,11 +17,13 @@ limitations under the License.
 package nodeup
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -33,7 +35,9 @@ import (
 	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/configserver"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -64,6 +68,8 @@ type NodeUpCommand struct {
 
 // Run is responsible for perform the nodeup process
 func (c *NodeUpCommand) Run(out io.Writer) error {
+	ctx := context.Background()
+
 	if c.ConfigLocation != "" {
 		config, err := vfs.Context.ReadFile(c.ConfigLocation)
 		if err != nil {
@@ -83,7 +89,17 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	}
 
 	var configBase vfs.Path
-	if fi.StringValue(c.config.ConfigBase) != "" {
+
+	// If we're using a config server instead of vfs, nodeConfig will hold our configuration
+	var nodeConfig *nodeup.NodeConfig
+
+	if c.config.ConfigServer != nil {
+		response, err := getNodeConfigFromServer(ctx, c.config.ConfigServer)
+		if err != nil {
+			return err
+		}
+		nodeConfig = response.NodeConfig
+	} else if fi.StringValue(c.config.ConfigBase) != "" {
 		var err error
 		configBase, err = vfs.Context.BuildVfsPath(*c.config.ConfigBase)
 		if err != nil {
@@ -102,11 +118,15 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 			return fmt.Errorf("cannot parse inferred ConfigBase %q: %v", basePath, err)
 		}
 	} else {
-		return fmt.Errorf("ConfigBase is required")
+		return fmt.Errorf("ConfigBase or ConfigServer is required")
 	}
 
 	c.cluster = &api.Cluster{}
-	{
+	if nodeConfig != nil {
+		if err := utils.YamlUnmarshal([]byte(nodeConfig.ClusterFullConfig), c.cluster); err != nil {
+			return fmt.Errorf("error parsing Cluster config response: %w", err)
+		}
+	} else {
 		clusterLocation := fi.StringValue(c.config.ClusterLocation)
 
 		var p vfs.Path
@@ -131,7 +151,12 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		}
 	}
 
-	if c.config.InstanceGroupName != "" {
+	if nodeConfig != nil {
+		c.instanceGroup = &api.InstanceGroup{}
+		if err := utils.YamlUnmarshal([]byte(nodeConfig.InstanceGroupConfig), c.instanceGroup); err != nil {
+			return fmt.Errorf("error parsing InstanceGroup config response: %v", err)
+		}
+	} else if c.config.InstanceGroupName != "" {
 		instanceGroupLocation := configBase.Join("instancegroup", c.config.InstanceGroupName)
 
 		c.instanceGroup = &api.InstanceGroup{}
@@ -183,7 +208,9 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 
 	var secretStore fi.SecretStore
 	var keyStore fi.Keystore
-	if c.cluster.Spec.SecretStore != "" {
+	if nodeConfig != nil {
+		modelContext.SecretStore = configserver.NewSecretStore(nodeConfig)
+	} else if c.cluster.Spec.SecretStore != "" {
 		klog.Infof("Building SecretStore at %q", c.cluster.Spec.SecretStore)
 		p, err := vfs.Context.BuildVfsPath(c.cluster.Spec.SecretStore)
 		if err != nil {
@@ -196,7 +223,9 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		return fmt.Errorf("SecretStore not set")
 	}
 
-	if c.cluster.Spec.KeyStore != "" {
+	if nodeConfig != nil {
+		modelContext.KeyStore = configserver.NewKeyStore(nodeConfig)
+	} else if c.cluster.Spec.KeyStore != "" {
 		klog.Infof("Building KeyStore at %q", c.cluster.Spec.KeyStore)
 		p, err := vfs.Context.BuildVfsPath(c.cluster.Spec.KeyStore)
 		if err != nil {
@@ -568,3 +597,43 @@ func loadKernelModules(context *model.NodeupModelContext) error {
 	// TODO: Add to /etc/modules-load.d/ ?
 	return nil
 }
+
+// getNodeConfigFromServer queries kops-controller for our node's configuration.
+func getNodeConfigFromServer(ctx context.Context, config *nodeup.ConfigServerOptions) (*nodeup.BootstrapResponse, error) {
+	var authenticator fi.Authenticator
+
+	switch api.CloudProviderID(config.CloudProvider) {
+	case api.CloudProviderAWS:
+		region, err := awsup.RegionFromMetadata(ctx)
+		if err != nil {
+			return nil, err
+		}
+		a, err := awsup.NewAWSAuthenticator(region)
+		if err != nil {
+			return nil, err
+		}
+		authenticator = a
+	default:
+		return nil, fmt.Errorf("unsupported cloud provider %s", config.CloudProvider)
+	}
+
+	client := &nodetasks.KopsBootstrapClient{
+		Authenticator: authenticator,
+	}
+
+	if config.CA != "" {
+		client.CA = []byte(config.CA)
+	}
+
+	u, err := url.Parse(config.Server)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse configuration server url %q: %w", config.Server, err)
+	}
+	client.BaseURL = *u
+
+	request := nodeup.BootstrapRequest{
+		APIVersion:        nodeup.BootstrapAPIVersion,
+		IncludeNodeConfig: true,
+	}
+	return client.QueryBootstrap(ctx, &request)
+}

upup/pkg/fi/nodeup/nodetasks/BUILD.bazel

@@ -26,7 +26,6 @@ go_library(
         "//pkg/backoff:go_default_library",
         "//pkg/kubeconfig:go_default_library",
         "//pkg/pki:go_default_library",
-        "//pkg/wellknownports:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/nodeup/cloudinit:go_default_library",
         "//upup/pkg/fi/nodeup/local:go_default_library",

upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go

@@ -19,6 +19,7 @@ package nodetasks
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
@@ -26,27 +27,23 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"net/url"
-	"strconv"
+	"path"
 
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/pki"
-	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-type BootstrapClient struct {
-	// Authenticator generates authentication credentials for requests.
-	Authenticator fi.Authenticator
-	// CA is the CA certificate for kops-controller.
-	CA []byte
+type BootstrapClientTask struct {
 	// Certs are the requested certificates.
 	Certs map[string]*BootstrapCert
 
-	client *http.Client
-	keys   map[string]*pki.PrivateKey
+	// Client holds the client wrapper for the kops-bootstrap protocol
+	Client *KopsBootstrapClient
+
+	keys map[string]*pki.PrivateKey
 }
 
 type BootstrapCert struct {
@@ -54,11 +51,11 @@ type BootstrapCert struct {
 	Key  *fi.TaskDependentResource
 }
 
-var _ fi.Task = &BootstrapClient{}
-var _ fi.HasName = &BootstrapClient{}
-var _ fi.HasDependencies = &BootstrapClient{}
+var _ fi.Task = &BootstrapClientTask{}
+var _ fi.HasName = &BootstrapClientTask{}
+var _ fi.HasDependencies = &BootstrapClientTask{}
 
-func (b *BootstrapClient) GetDependencies(tasks map[string]fi.Task) []fi.Task {
+func (b *BootstrapClientTask) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 	// BootstrapClient depends on the protokube service to ensure gossip DNS
 	var deps []fi.Task
 	for _, v := range tasks {
@@ -69,16 +66,18 @@ func (b *BootstrapClient) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 	return deps
 }
 
-func (b *BootstrapClient) GetName() *string {
+func (b *BootstrapClientTask) GetName() *string {
 	name := "BootstrapClient"
 	return &name
 }
 
-func (b *BootstrapClient) String() string {
-	return "BootstrapClient"
+func (b *BootstrapClientTask) String() string {
+	return "BootstrapClientTask"
 }
 
-func (b *BootstrapClient) Run(c *fi.Context) error {
+func (b *BootstrapClientTask) Run(c *fi.Context) error {
+	ctx := context.TODO()
+
 	req := nodeup.BootstrapRequest{
 		APIVersion: nodeup.BootstrapAPIVersion,
 		Certs:      map[string]string{},
@@ -109,7 +108,7 @@ func (b *BootstrapClient) Run(c *fi.Context) error {
 		req.Certs[name] = string(pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: pkData}))
 	}
 
-	resp, err := b.queryBootstrap(c, &req)
+	resp, err := b.Client.QueryBootstrap(ctx, &req)
 	if err != nil {
 		return err
 	}
@@ -129,12 +128,24 @@ func (b *BootstrapClient) Run(c *fi.Context) error {
 	return nil
 }
 
-func (b *BootstrapClient) queryBootstrap(c *fi.Context, req *nodeup.BootstrapRequest) (*nodeup.BootstrapResponse, error) {
-	if b.client == nil {
+type KopsBootstrapClient struct {
+	// Authenticator generates authentication credentials for requests.
+	Authenticator fi.Authenticator
+	// CA is the CA certificate for kops-controller.
+	CA []byte
+
+	// BaseURL is the base URL for the server
+	BaseURL url.URL
+
+	httpClient *http.Client
+}
+
+func (b *KopsBootstrapClient) QueryBootstrap(ctx context.Context, req *nodeup.BootstrapRequest) (*nodeup.BootstrapResponse, error) {
+	if b.httpClient == nil {
 		certPool := x509.NewCertPool()
 		certPool.AppendCertsFromPEM(b.CA)
 
-		b.client = &http.Client{
+		b.httpClient = &http.Client{
 			Transport: &http.Transport{
 				TLSClientConfig: &tls.Config{
 					RootCAs:    certPool,
@@ -149,12 +160,9 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req *nodeup.BootstrapReq
 		return nil, err
 	}
 
-	bootstrapUrl := url.URL{
-		Scheme: "https",
-		Host:   net.JoinHostPort("kops-controller.internal."+c.Cluster.ObjectMeta.Name, strconv.Itoa(wellknownports.KopsControllerPort)),
-		Path:   "/bootstrap",
-	}
-	httpReq, err := http.NewRequest("POST", bootstrapUrl.String(), bytes.NewReader(reqBytes))
+	bootstrapURL := b.BaseURL
+	bootstrapURL.Path = path.Join(bootstrapURL.Path, "/bootstrap")
+	httpReq, err := http.NewRequestWithContext(ctx, "POST", bootstrapURL.String(), bytes.NewReader(reqBytes))
 	if err != nil {
 		return nil, err
 	}
@@ -166,7 +174,7 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req *nodeup.BootstrapReq
 	}
 	httpReq.Header.Set("Authorization", token)
 
-	resp, err := b.client.Do(httpReq)
+	resp, err := b.httpClient.Do(httpReq)
 	if err != nil {
 		return nil, err
 	}

upup/pkg/fi/nodeup/nodetasks/service.go

@@ -75,7 +75,7 @@ func (p *Service) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 		switch v := v.(type) {
 		case *Package, *UpdatePackages, *UserTask, *GroupTask, *Chattr, *BindMount, *Archive:
 			deps = append(deps, v)
-		case *Service, *LoadImageTask, *IssueCert, *BootstrapClient, *KubeConfig:
+		case *Service, *LoadImageTask, *IssueCert, *BootstrapClientTask, *KubeConfig:
 			// ignore
 		case *File:
 			if len(v.BeforeServices) > 0 {