Expose JWKS via a feature-flag

When the PublicJWKS feature-flag is set, we expose the apiserver JWKS
document publicly (including enabling anonymous access).  This is a
stepping stone to a more hardened configuration where we copy the JWKS
document to S3/GCS/etc.

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

cloudmock/aws/mockiam/oidcprovider.go

@@ -17,6 +17,7 @@ limitations under the License.
 package mockiam
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -48,11 +49,28 @@ func (m *MockIAM) ListOpenIDConnectProvidersRequest(*iam.ListOpenIDConnectProvid
 	panic("Not implemented")
 }
 
-func (m *MockIAM) GetOpenIDConnectProviderWithContext(aws.Context, *iam.GetOpenIDConnectProviderInput, ...request.Option) (*iam.GetOpenIDConnectProviderOutput, error) {
+func (m *MockIAM) GetOpenIDConnectProviderWithContext(ctx aws.Context, request *iam.GetOpenIDConnectProviderInput, options ...request.Option) (*iam.GetOpenIDConnectProviderOutput, error) {
-	panic("Not implemented")
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	arn := aws.StringValue(request.OpenIDConnectProviderArn)
+
+	provider := m.OIDCProviders[arn]
+	if provider == nil {
+		return nil, fmt.Errorf("OpenIDConnectProvider with arn=%q not found", arn)
+	}
+
+	response := &iam.GetOpenIDConnectProviderOutput{
+		ClientIDList:   provider.ClientIDList,
+		CreateDate:     provider.CreateDate,
+		ThumbprintList: provider.ThumbprintList,
+		Url:            provider.Url,
+	}
+	return response, nil
 }
-func (m *MockIAM) GetOpenIDConnectProvider(*iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
+
-	panic("Not implemented")
+func (m *MockIAM) GetOpenIDConnectProvider(request *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) {
+	return m.GetOpenIDConnectProviderWithContext(context.Background(), request)
 }
 
 func (m *MockIAM) GetOpenIDConnectProviderRequest(*iam.GetOpenIDConnectProviderInput) (*request.Request, *iam.GetOpenIDConnectProviderOutput) {

cmd/kops/integration_test.go

@@ -61,8 +61,10 @@ type integrationTest struct {
 	launchConfiguration bool
 	lifecycleOverrides  []string
 	sshKey              bool
-	jsonOutput          bool
+	// caKey is true if we should use a provided ca.crt & ca.key as our CA
-	bastionUserData     bool
+	caKey           bool
+	jsonOutput      bool
+	bastionUserData bool
 }
 
 func newIntegrationTest(clusterName, srcDir string) *integrationTest {
@@ -91,6 +93,13 @@ func (i *integrationTest) withoutSSHKey() *integrationTest {
 	return i
 }
 
+// withCAKey indicates that we should use a fixed ca.crt & ca.key from the source directory as our CA.
+// This is needed when the CA is exposed, for example when using AWS WebIdentity federation.
+func (i *integrationTest) withCAKey() *integrationTest {
+	i.caKey = true
+	return i
+}
+
 func (i *integrationTest) withoutPolicies() *integrationTest {
 	i.expectPolicies = false
 	return i
@@ -314,15 +323,27 @@ func TestContainerdCloudformation(t *testing.T) {
 // TestLaunchConfigurationASG tests ASGs using launch configurations instead of launch templates
 func TestLaunchConfigurationASG(t *testing.T) {
 	featureflag.ParseFlags("-EnableLaunchTemplates")
-	unsetFeaureFlag := func() {
+	unsetFeatureFlags := func() {
 		featureflag.ParseFlags("+EnableLaunchTemplates")
 	}
-	defer unsetFeaureFlag()
+	defer unsetFeatureFlags()
 
 	newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestTerraformAWS(t)
 	newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestCloudformation(t)
 }
 
+// TestJWKS runs a simple configuration, but with PublicJWKS enabled
+func TestPublicJWKS(t *testing.T) {
+	featureflag.ParseFlags("+PublicJWKS")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-PublicJWKS")
+	}
+	defer unsetFeatureFlags()
+
+	// We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
+	newIntegrationTest("minimal.example.com", "public-jwks").withCAKey().runTestTerraformAWS(t)
+}
+
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
 	ctx := context.Background()
 
@@ -364,7 +385,19 @@ func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarn
 
 		err := RunCreateSecretPublicKey(ctx, factory, &stdout, options)
 		if err != nil {
-			t.Fatalf("error running %q create: %v", inputYAML, err)
+			t.Fatalf("error running %q create public key: %v", inputYAML, err)
+		}
+	}
+
+	if i.caKey {
+		options := &CreateSecretCaCertOptions{}
+		options.ClusterName = i.clusterName
+		options.CaPrivateKeyPath = path.Join(i.srcDir, "ca.key")
+		options.CaCertPath = path.Join(i.srcDir, "ca.crt")
+
+		err := RunCreateSecretCaCert(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create CA keypair: %v", inputYAML, err)
 		}
 	}
 

nodeup/pkg/model/kube_apiserver.go

@@ -287,8 +287,18 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 // buildPod is responsible for generating the kube-apiserver pod and thus manifest file
 func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	kubeAPIServer := b.Cluster.Spec.KubeAPIServer
+
 	// TODO pass the public key instead. We would first need to segregate the secrets better.
 	kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.key"))
+
+	// Set the signing key if we're using Service Account Token VolumeProjection
+	if kubeAPIServer.ServiceAccountSigningKeyFile == nil {
+		if fi.StringValue(kubeAPIServer.ServiceAccountIssuer) != "" {
+			s := filepath.Join(b.PathSrvKubernetes(), "service-account.key")
+			kubeAPIServer.ServiceAccountSigningKeyFile = &s
+		}
+	}
+
 	kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 	kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt")
 	kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key")

pkg/featureflag/featureflag.go

@@ -94,6 +94,8 @@ var (
 	LegacyIAM = New("LegacyIAM", Bool(false))
 	// ClusterAddons activates experimental cluster-addons support
 	ClusterAddons = New("ClusterAddons", Bool(false))
+	// PublicJWKS enables public jwks access.  This is generally not as secure as republishing.
+	PublicJWKS = New("PublicJWKS", Bool(false))
 )
 
 // FeatureFlag defines a feature flag

pkg/model/awsmodel/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "api_loadbalancer.go",
         "autoscalinggroup.go",
         "context.go",
+        "oidc_provider.go",
     ],
     importpath = "k8s.io/kops/pkg/model/awsmodel",
     visibility = ["//visibility:public"],
@@ -15,9 +16,11 @@ go_library(
         "//pkg/featureflag:go_default_library",
         "//pkg/model:go_default_library",
         "//pkg/model/defaults:go_default_library",
+        "//pkg/model/iam:go_default_library",
         "//pkg/model/spotinstmodel:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awstasks:go_default_library",
+        "//upup/pkg/fi/fitasks:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",

pkg/model/awsmodel/oidc_provider.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsmodel
+
+import (
+	"fmt"
+
+	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/model/iam"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
+	"k8s.io/kops/upup/pkg/fi/fitasks"
+)
+
+// OIDCProviderBuilder configures IAM OIDC Provider
+type OIDCProviderBuilder struct {
+	*model.KopsModelContext
+
+	KeyStore  fi.CAStore
+	Lifecycle *fi.Lifecycle
+}
+
+var _ fi.ModelBuilder = &OIDCProviderBuilder{}
+
+const (
+	defaultAudience = "amazonaws.com"
+)
+
+func (b *OIDCProviderBuilder) Build(c *fi.ModelBuilderContext) error {
+	var thumbprints []fi.Resource
+	var issuerURL string
+
+	if featureflag.PublicJWKS.Enabled() {
+		serviceAccountIssuer, err := iam.ServiceAccountIssuer(b.ClusterName(), &b.Cluster.Spec)
+		if err != nil {
+			return err
+		}
+		issuerURL = serviceAccountIssuer
+
+		caTaskObject, found := c.Tasks["Keypair/ca"]
+		if !found {
+			return fmt.Errorf("keypair/ca task not found")
+		}
+
+		caTask := caTaskObject.(*fitasks.Keypair)
+		fingerprint := caTask.CertificateSHA1Fingerprint()
+
+		thumbprints = []fi.Resource{fingerprint}
+	}
+
+	if issuerURL == "" {
+		return nil
+	}
+
+	c.AddTask(&awstasks.IAMOIDCProvider{
+		Name:        fi.String(b.ClusterName()),
+		Lifecycle:   b.Lifecycle,
+		URL:         fi.String(issuerURL),
+		ClientIDs:   []*string{fi.String(defaultAudience)},
+		Thumbprints: thumbprints,
+	})
+
+	return nil
+}

pkg/model/components/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "containerd.go",
         "context.go",
         "defaults.go",
+        "discovery.go",
         "docker.go",
         "etcd.go",
         "kubecontrollermanager.go",
@@ -24,7 +25,9 @@ go_library(
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/kops/util:go_default_library",
         "//pkg/assets:go_default_library",
+        "//pkg/featureflag:go_default_library",
         "//pkg/k8sversion:go_default_library",
+        "//pkg/model/iam:go_default_library",
         "//pkg/wellknownports:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",

pkg/model/components/apiserver.go

@@ -22,6 +22,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
 
@@ -208,6 +209,12 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
 	// We make sure to disable AnonymousAuth
 	c.AnonymousAuth = fi.Bool(false)
 
+	// For the development of JWKS functionality, re-enable anonymous auth for PublicJWKS
+	// (this likely isn't a production-suitable configuration, currently)
+	if featureflag.PublicJWKS.Enabled() {
+		c.AnonymousAuth = fi.Bool(true)
+	}
+
 	if b.IsKubernetesGTE("1.17") {
 		// We query via the kube-apiserver-healthcheck proxy, which listens on port 3990
 		c.InsecurePort = 0

pkg/model/components/discovery.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package components
+
+import (
+	"strings"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/model/iam"
+	"k8s.io/kops/upup/pkg/fi/loader"
+)
+
+// DiscoveryOptionsBuilder adds options for identity discovery to the model (mostly kube-apiserver)
+type DiscoveryOptionsBuilder struct {
+	*OptionsContext
+}
+
+var _ loader.OptionsBuilder = &DiscoveryOptionsBuilder{}
+
+func (b *DiscoveryOptionsBuilder) BuildOptions(o interface{}) error {
+	clusterSpec := o.(*kops.ClusterSpec)
+
+	useJWKS := featureflag.PublicJWKS.Enabled()
+	if !useJWKS {
+		return nil
+	}
+
+	if clusterSpec.KubeAPIServer == nil {
+		clusterSpec.KubeAPIServer = &kops.KubeAPIServerConfig{}
+	}
+
+	kubeAPIServer := clusterSpec.KubeAPIServer
+
+	if kubeAPIServer.FeatureGates == nil {
+		kubeAPIServer.FeatureGates = make(map[string]string)
+	}
+	kubeAPIServer.FeatureGates["ServiceAccountIssuerDiscovery"] = "true"
+
+	if len(kubeAPIServer.APIAudiences) == 0 {
+		kubeAPIServer.APIAudiences = []string{"kubernetes.svc.default"}
+	}
+
+	if kubeAPIServer.ServiceAccountIssuer == nil {
+		serviceAccountIssuer, err := iam.ServiceAccountIssuer(b.ClusterName, clusterSpec)
+		if err != nil {
+			return err
+		}
+		kubeAPIServer.ServiceAccountIssuer = &serviceAccountIssuer
+	}
+
+	if kubeAPIServer.ServiceAccountJWKSURI == nil {
+		jwksURL := *kubeAPIServer.ServiceAccountIssuer
+		jwksURL = strings.TrimSuffix(jwksURL, "/") + "/openid/v1/jwks"
+
+		kubeAPIServer.ServiceAccountJWKSURI = &jwksURL
+	}
+
+	// We set apiserver ServiceAccountKey and ServiceAccountSigningKeyFile in nodeup
+
+	return nil
+}

pkg/model/iam/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "iam_builder.go",
+        "pod_roles.go",
         "types.go",
     ],
     importpath = "k8s.io/kops/pkg/model/iam",
@@ -11,6 +12,7 @@ go_library(
     deps = [
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/kops/model:go_default_library",
+        "//pkg/featureflag:go_default_library",
         "//pkg/util/stringorslice:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awstasks:go_default_library",

pkg/model/iam/pod_roles.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iam
+
+import (
+	"fmt"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/featureflag"
+)
+
+// ServiceAccountIssuer determines the issuer in the ServiceAccount JWTs
+func ServiceAccountIssuer(clusterName string, clusterSpec *kops.ClusterSpec) (string, error) {
+	if featureflag.PublicJWKS.Enabled() {
+		return "https://api." + clusterName, nil
+	}
+
+	return "", fmt.Errorf("ServiceAcccountIssuer not (currently) supported without PublicJWKS")
+}

tests/integration/update_cluster/public-jwks/README.md

@@ -0,0 +1,9 @@
+Simple test of (experimental) JWKS functionality
+
+We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration.
+
+ca.crt & ca.key generated with:
+
+`openssl req -new -newkey rsa:512 -days 3650 -nodes -x509 -subj "/CN=kubernetes" -keyout ca.key -out ca.crt`
+
+

tests/integration/update_cluster/public-jwks/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBgTCCASugAwIBAgIUZrxLCo6MlBXbjRWuIBXdlRkM2EcwDQYJKoZIhvcNAQEL
+BQAwFTETMBEGA1UEAwwKa3ViZXJuZXRlczAeFw0yMDA4MTUyMTM3NDhaFw0zMDA4
+MTMyMTM3NDhaMBUxEzARBgNVBAMMCmt1YmVybmV0ZXMwXDANBgkqhkiG9w0BAQEF
+AANLADBIAkEA5eJVxg/iR9zq2wQrk2VjdavGYiPu1Q0cmNb4LvItHBO0eiSVA7EV
+D/7qAgnB13ASaQHLMuG50qK3wihMJC9/6QIDAQABo1MwUTAdBgNVHQ4EFgQU4/Jf
+ZYu5ziuhZRnpcxvDOlYGA+4wHwYDVR0jBBgwFoAU4/JfZYu5ziuhZRnpcxvDOlYG
+A+4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEHceMm6tpH6Yc+H
+5uu5wY8Q4pmYJt+HOkIpoXO1KD4/8h90y6XY8Z0Nu3dOZSwBSCWChrYAIndtzJfC
+PtQHwNM=
+-----END CERTIFICATE-----

tests/integration/update_cluster/public-jwks/ca.key

@@ -0,0 +1,10 @@
+-----BEGIN PRIVATE KEY-----
+MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA5eJVxg/iR9zq2wQr
+k2VjdavGYiPu1Q0cmNb4LvItHBO0eiSVA7EVD/7qAgnB13ASaQHLMuG50qK3wihM
+JC9/6QIDAQABAkEAug/7RJfOmkOggyxY6LADVFZ39y8GO8KlBr/XmIfDIxj20yIG
+W2SmoSGPqoWDpr8G2LUSVrdaQ9ZyDqG0AqUN0QIhAPx5JQRoRDo2hiS+Ioaty/NA
+7/iInYFkS5hMvud1QSKDAiEA6RhpLIFZbLAoof6/fdIUy7QWU1UHJ6PKq/3qpR7u
+mCMCIQCVmHKGmgFTPNtfCgoLIw+louSNruUktfjU1SSIoMFnYQIgLxR8Ib4ahsZp
+3pZqrQoioyZDoB87a7k8dVK68xD1VgsCIHFjAVxGmS2MgT80UjwPNs9XkT5WOpoR
+BzhivO3D3oOn
+-----END PRIVATE KEY-----

tests/integration/update_cluster/public-jwks/data/aws_iam_role_masters.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/public-jwks/data/aws_iam_role_nodes.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -0,0 +1,170 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DescribeVolumesModifications",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Resource": [
+        "*"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Resource": [
+        "*"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}

tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_nodes.minimal.example.com_policy

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}

tests/integration/update_cluster/public-jwks/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/public-jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -0,0 +1,330 @@
+#!/bin/bash
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=deff1179f80bd65232af30c252a2b5f390b160bbfe9990637c0ec88d76cdd647
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-arm64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=300bfade024ede4fe8996c9ee6393e97463bea4eaffd0876310710dcdebbf8d3
+
+export AWS_REGION=us-test-1
+
+
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig: null
+containerRuntime: docker
+containerd:
+  skipInstall: true
+docker:
+  ipMasq: false
+  ipTables: false
+  logDriver: json-file
+  logLevel: info
+  logOpt:
+  - max-size=10m
+  - max-file=5
+  storage: overlay2,overlay,aufs
+  version: 18.06.3
+encryptionConfig: null
+etcdClusters:
+  events:
+    version: 3.3.10
+  main:
+    version: 3.3.10
+kubeAPIServer:
+  allowPrivileged: true
+  anonymousAuth: true
+  apiAudiences:
+  - kubernetes.svc.default
+  apiServerCount: 1
+  authorizationMode: AlwaysAllow
+  bindAddress: 0.0.0.0
+  cloudProvider: aws
+  enableAdmissionPlugins:
+  - NamespaceLifecycle
+  - LimitRanger
+  - ServiceAccount
+  - PersistentVolumeLabel
+  - DefaultStorageClass
+  - DefaultTolerationSeconds
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
+  - NodeRestriction
+  - ResourceQuota
+  etcdServers:
+  - http://127.0.0.1:4001
+  etcdServersOverrides:
+  - /events#http://127.0.0.1:4002
+  featureGates:
+    ServiceAccountIssuerDiscovery: "true"
+  image: k8s.gcr.io/kube-apiserver:v1.14.0
+  insecureBindAddress: 127.0.0.1
+  insecurePort: 8080
+  kubeletPreferredAddressTypes:
+  - InternalIP
+  - Hostname
+  - ExternalIP
+  logLevel: 2
+  requestheaderAllowedNames:
+  - aggregator
+  requestheaderExtraHeaderPrefixes:
+  - X-Remote-Extra-
+  requestheaderGroupHeaders:
+  - X-Remote-Group
+  requestheaderUsernameHeaders:
+  - X-Remote-User
+  securePort: 443
+  serviceAccountIssuer: https://api.minimal.example.com
+  serviceAccountJWKSURI: https://api.minimal.example.com/openid/v1/jwks
+  serviceClusterIPRange: 100.64.0.0/13
+  storageBackend: etcd3
+kubeControllerManager:
+  allocateNodeCIDRs: true
+  attachDetachReconcileSyncPeriod: 1m0s
+  cloudProvider: aws
+  clusterCIDR: 100.96.0.0/11
+  clusterName: minimal.example.com
+  configureCloudRoutes: true
+  image: k8s.gcr.io/kube-controller-manager:v1.14.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+  useServiceAccountCredentials: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.14.0
+  logLevel: 2
+kubeScheduler:
+  image: k8s.gcr.io/kube-scheduler:v1.14.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    ExperimentalCriticalPodAnnotation: "true"
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginMTU: 9001
+  networkPluginName: kubenet
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podInfraContainerImage: k8s.gcr.io/pause:3.2
+  podManifestPath: /etc/kubernetes/manifests
+masterKubelet:
+  anonymousAuth: false
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    ExperimentalCriticalPodAnnotation: "true"
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginMTU: 9001
+  networkPluginName: kubenet
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podInfraContainerImage: k8s.gcr.io/pause:3.2
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet
+  - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl
+  - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz
+  arm64:
+  - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet
+  - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl
+  - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz
+ClusterName: minimal.example.com
+ConfigBase: memfs://clusters.example.com/minimal.example.com
+InstanceGroupName: master-us-test-1a
+InstanceGroupRole: Master
+KubeletConfig:
+  anonymousAuth: false
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    ExperimentalCriticalPodAnnotation: "true"
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginMTU: 9001
+  networkPluginName: kubenet
+  nodeLabels:
+    kubernetes.io/role: master
+    node-role.kubernetes.io/master: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podInfraContainerImage: k8s.gcr.io/pause:3.2
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+channels:
+- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+etcdManifests:
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/public-jwks/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -0,0 +1,237 @@
+#!/bin/bash
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=deff1179f80bd65232af30c252a2b5f390b160bbfe9990637c0ec88d76cdd647
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-arm64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=300bfade024ede4fe8996c9ee6393e97463bea4eaffd0876310710dcdebbf8d3
+
+export AWS_REGION=us-test-1
+
+
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig: null
+containerRuntime: docker
+containerd:
+  skipInstall: true
+docker:
+  ipMasq: false
+  ipTables: false
+  logDriver: json-file
+  logLevel: info
+  logOpt:
+  - max-size=10m
+  - max-file=5
+  storage: overlay2,overlay,aufs
+  version: 18.06.3
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.14.0
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    ExperimentalCriticalPodAnnotation: "true"
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginMTU: 9001
+  networkPluginName: kubenet
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podInfraContainerImage: k8s.gcr.io/pause:3.2
+  podManifestPath: /etc/kubernetes/manifests
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet
+  - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl
+  - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz
+  arm64:
+  - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet
+  - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl
+  - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz
+ClusterName: minimal.example.com
+ConfigBase: memfs://clusters.example.com/minimal.example.com
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+KubeletConfig:
+  anonymousAuth: false
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    ExperimentalCriticalPodAnnotation: "true"
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginMTU: 9001
+  networkPluginName: kubenet
+  nodeLabels:
+    kubernetes.io/role: node
+    node-role.kubernetes.io/node: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podInfraContainerImage: k8s.gcr.io/pause:3.2
+  podManifestPath: /etc/kubernetes/manifests
+channels:
+- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/public-jwks/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/public-jwks/in-v1alpha2.yaml

@@ -0,0 +1,80 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.14.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+
+

tests/integration/update_cluster/public-jwks/kubernetes.tf

@@ -0,0 +1,548 @@
+locals {
+  cluster_name                 = "minimal.example.com"
+  master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
+  master_security_group_ids    = [aws_security_group.masters-minimal-example-com.id]
+  masters_role_arn             = aws_iam_role.masters-minimal-example-com.arn
+  masters_role_name            = aws_iam_role.masters-minimal-example-com.name
+  node_autoscaling_group_ids   = [aws_autoscaling_group.nodes-minimal-example-com.id]
+  node_security_group_ids      = [aws_security_group.nodes-minimal-example-com.id]
+  node_subnet_ids              = [aws_subnet.us-test-1a-minimal-example-com.id]
+  nodes_role_arn               = aws_iam_role.nodes-minimal-example-com.arn
+  nodes_role_name              = aws_iam_role.nodes-minimal-example-com.name
+  region                       = "us-test-1"
+  route_table_public_id        = aws_route_table.minimal-example-com.id
+  subnet_us-test-1a_id         = aws_subnet.us-test-1a-minimal-example-com.id
+  vpc_cidr_block               = aws_vpc.minimal-example-com.cidr_block
+  vpc_id                       = aws_vpc.minimal-example-com.id
+}
+
+output "cluster_name" {
+  value = "minimal.example.com"
+}
+
+output "master_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
+}
+
+output "master_security_group_ids" {
+  value = [aws_security_group.masters-minimal-example-com.id]
+}
+
+output "masters_role_arn" {
+  value = aws_iam_role.masters-minimal-example-com.arn
+}
+
+output "masters_role_name" {
+  value = aws_iam_role.masters-minimal-example-com.name
+}
+
+output "node_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.nodes-minimal-example-com.id]
+}
+
+output "node_security_group_ids" {
+  value = [aws_security_group.nodes-minimal-example-com.id]
+}
+
+output "node_subnet_ids" {
+  value = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+output "nodes_role_arn" {
+  value = aws_iam_role.nodes-minimal-example-com.arn
+}
+
+output "nodes_role_name" {
+  value = aws_iam_role.nodes-minimal-example-com.name
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "route_table_public_id" {
+  value = aws_route_table.minimal-example-com.id
+}
+
+output "subnet_us-test-1a_id" {
+  value = aws_subnet.us-test-1a-minimal-example-com.id
+}
+
+output "vpc_cidr_block" {
+  value = aws_vpc.minimal-example-com.cidr_block
+}
+
+output "vpc_id" {
+  value = aws_vpc.minimal-example-com.id
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id
+    version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version
+  }
+  max_size            = 1
+  metrics_granularity = "1Minute"
+  min_size            = 1
+  name                = "master-us-test-1a.masters.minimal.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "minimal.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "master-us-test-1a.masters.minimal.example.com"
+  }
+  tag {
+    key                 = "k8s.io/role/master"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "master-us-test-1a"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/minimal.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+resource "aws_autoscaling_group" "nodes-minimal-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.nodes-minimal-example-com.id
+    version = aws_launch_template.nodes-minimal-example-com.latest_version
+  }
+  max_size            = 2
+  metrics_granularity = "1Minute"
+  min_size            = 2
+  name                = "nodes.minimal.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "minimal.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "nodes.minimal.example.com"
+  }
+  tag {
+    key                 = "k8s.io/role/node"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "nodes"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/minimal.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  size              = 20
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.etcd-events.minimal.example.com"
+    "k8s.io/etcd/events"                        = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                        = "1"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  type = "gp2"
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  size              = 20
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.etcd-main.minimal.example.com"
+    "k8s.io/etcd/main"                          = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                        = "1"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  type = "gp2"
+}
+
+resource "aws_iam_instance_profile" "masters-minimal-example-com" {
+  name = "masters.minimal.example.com"
+  role = aws_iam_role.masters-minimal-example-com.name
+}
+
+resource "aws_iam_instance_profile" "nodes-minimal-example-com" {
+  name = "nodes.minimal.example.com"
+  role = aws_iam_role.nodes-minimal-example-com.name
+}
+
+resource "aws_iam_openid_connect_provider" "minimal-example-com" {
+  client_id_list  = ["amazonaws.com"]
+  thumbprint_list = ["d89b37ccc0b574f3e40051ea08a7b60a9db11924"]
+  url             = "https://api.minimal.example.com"
+}
+
+resource "aws_iam_role_policy" "masters-minimal-example-com" {
+  name   = "masters.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy")
+  role   = aws_iam_role.masters-minimal-example-com.name
+}
+
+resource "aws_iam_role_policy" "nodes-minimal-example-com" {
+  name   = "nodes.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy")
+  role   = aws_iam_role.nodes-minimal-example-com.name
+}
+
+resource "aws_iam_role" "masters-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy")
+  name               = "masters.minimal.example.com"
+}
+
+resource "aws_iam_role" "nodes-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy")
+  name               = "nodes.minimal.example.com"
+}
+
+resource "aws_internet_gateway" "minimal-example-com" {
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      volume_size           = 64
+      volume_type           = "gp2"
+    }
+  }
+  block_device_mappings {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.masters-minimal-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "m3.medium"
+  key_name      = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  name_prefix = "master-us-test-1a.masters.minimal.example.com-"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.masters-minimal-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                         = "minimal.example.com"
+      "Name"                                      = "master-us-test-1a.masters.minimal.example.com"
+      "k8s.io/role/master"                        = "1"
+      "kops.k8s.io/instancegroup"                 = "master-us-test-1a"
+      "kubernetes.io/cluster/minimal.example.com" = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                         = "minimal.example.com"
+      "Name"                                      = "master-us-test-1a.masters.minimal.example.com"
+      "k8s.io/role/master"                        = "1"
+      "kops.k8s.io/instancegroup"                 = "master-us-test-1a"
+      "kubernetes.io/cluster/minimal.example.com" = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "master-us-test-1a.masters.minimal.example.com"
+    "k8s.io/role/master"                        = "1"
+    "kops.k8s.io/instancegroup"                 = "master-us-test-1a"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data")
+}
+
+resource "aws_launch_template" "nodes-minimal-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      volume_size           = 128
+      volume_type           = "gp2"
+    }
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.nodes-minimal-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "t2.medium"
+  key_name      = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  name_prefix = "nodes.minimal.example.com-"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.nodes-minimal-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                         = "minimal.example.com"
+      "Name"                                      = "nodes.minimal.example.com"
+      "k8s.io/role/node"                          = "1"
+      "kops.k8s.io/instancegroup"                 = "nodes"
+      "kubernetes.io/cluster/minimal.example.com" = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                         = "minimal.example.com"
+      "Name"                                      = "nodes.minimal.example.com"
+      "k8s.io/role/node"                          = "1"
+      "kops.k8s.io/instancegroup"                 = "nodes"
+      "kubernetes.io/cluster/minimal.example.com" = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "nodes.minimal.example.com"
+    "k8s.io/role/node"                          = "1"
+    "kops.k8s.io/instancegroup"                 = "nodes"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data")
+}
+
+resource "aws_route_table_association" "us-test-1a-minimal-example-com" {
+  route_table_id = aws_route_table.minimal-example-com.id
+  subnet_id      = aws_subnet.us-test-1a-minimal-example-com.id
+}
+
+resource "aws_route_table" "minimal-example-com" {
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+    "kubernetes.io/kops/role"                   = "public"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_route" "route-0-0-0-0--0" {
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = aws_internet_gateway.minimal-example-com.id
+  route_table_id         = aws_route_table.minimal-example-com.id
+}
+
+resource "aws_security_group_rule" "all-master-to-master" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "all-master-to-node" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-minimal-example-com.id
+  source_security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "all-node-to-node" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 443
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 443
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "master-egress" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "node-egress" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
+  from_port                = 1
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 2379
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  from_port                = 2382
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 4000
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" {
+  from_port                = 4003
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "node-to-master-udp-1-65535" {
+  from_port                = 1
+  protocol                 = "udp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group" "masters-minimal-example-com" {
+  description = "Security group for masters"
+  name        = "masters.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "masters.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_security_group" "nodes-minimal-example-com" {
+  description = "Security group for nodes"
+  name        = "nodes.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "nodes.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_subnet" "us-test-1a-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  cidr_block        = "172.20.32.0/19"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.minimal.example.com"
+    "SubnetType"                                = "Public"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+    "kubernetes.io/role/elb"                    = "1"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_vpc_dhcp_options_association" "minimal-example-com" {
+  dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id
+  vpc_id          = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_vpc_dhcp_options" "minimal-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc" "minimal-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+terraform {
+  required_version = ">= 0.12.0"
+}

upup/models/bindata.go

@@ -1,6 +1,7 @@
 // Code generated for package models by go-bindata DO NOT EDIT. (@generated)
 // sources:
 // upup/models/cloudup/resources/addons/OWNERS
+// upup/models/cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template
 // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.10.yaml.template
 // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template
 // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml
@@ -136,6 +137,37 @@ func cloudupResourcesAddonsOwners() (*asset, error) {
 	return a, nil
 }
 
+var _cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: anonymous-access.addons.k8s.io
+  name: anonymous:service-account-issuer-discovery
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:service-account-issuer-discovery
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:anonymous`)
+
+func cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplateBytes() ([]byte, error) {
+	return _cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate, nil
+}
+
+func cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate() (*asset, error) {
+	bytes, err := cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplateBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate = []byte(`---
 apiVersion: extensions/v1beta1
 kind: DaemonSet
@@ -20361,7 +20393,8 @@ func AssetNames() []string {
 
 // _bindata is a table, holding each asset generator, mapped to its name.
 var _bindata = map[string]func() (*asset, error){
-	"cloudup/resources/addons/OWNERS":                                                                     cloudupResourcesAddonsOwners,
+	"cloudup/resources/addons/OWNERS": cloudupResourcesAddonsOwners,
+	"cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template":            cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate,
 	"cloudup/resources/addons/authentication.aws/k8s-1.10.yaml.template":                                  cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate,
 	"cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template":                                  cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate,
 	"cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml":                                       cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml,
@@ -20471,6 +20504,9 @@ var _bintree = &bintree{nil, map[string]*bintree{
 		"resources": {nil, map[string]*bintree{
 			"addons": {nil, map[string]*bintree{
 				"OWNERS": {cloudupResourcesAddonsOwners, map[string]*bintree{}},
+				"anonymous-issuer-discovery.addons.k8s.io": {nil, map[string]*bintree{
+					"k8s-1.16.yaml.template": {cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate, map[string]*bintree{}},
+				}},
 				"authentication.aws": {nil, map[string]*bintree{
 					"k8s-1.10.yaml.template": {cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate, map[string]*bintree{}},
 					"k8s-1.12.yaml.template": {cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate, map[string]*bintree{}},

upup/models/cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: anonymous-access.addons.k8s.io
+  name: anonymous:service-account-issuer-discovery
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:service-account-issuer-discovery
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:anonymous

upup/pkg/fi/cloudup/BUILD.bazel

@@ -117,6 +117,7 @@ go_test(
         "//pkg/assets:go_default_library",
         "//pkg/client/simple/vfsclientset:go_default_library",
         "//pkg/diff:go_default_library",
+        "//pkg/featureflag:go_default_library",
         "//pkg/kopscodecs:go_default_library",
         "//pkg/model:go_default_library",
         "//pkg/model/iam:go_default_library",

upup/pkg/fi/cloudup/apply_cluster.go

@@ -524,6 +524,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 
 			l.Builders = append(l.Builders,
 				&model.IAMModelBuilder{KopsModelContext: modelContext, Lifecycle: &securityLifecycle},
+				&awsmodel.OIDCProviderBuilder{KopsModelContext: modelContext, Lifecycle: &securityLifecycle, KeyStore: keyStore},
 			)
 		case kops.CloudProviderDO:
 			doModelContext := &domodel.DOModelContext{

upup/pkg/fi/cloudup/awstasks/iamoidcprovider.go

@@ -19,6 +19,7 @@ package awstasks
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
@@ -33,12 +34,13 @@ import (
 type IAMOIDCProvider struct {
 	Lifecycle *fi.Lifecycle
 
-	ARN         *string
 	ClientIDs   []*string
-	Thumbprints []*string
+	Thumbprints []fi.Resource
 	URL         *string
 
 	Name *string
+
+	arn *string
 }
 
 var _ fi.CompareWithID = &IAMOIDCProvider{}
@@ -64,13 +66,29 @@ func (e *IAMOIDCProvider) Find(c *fi.Context) (*IAMOIDCProvider, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error describing oidc provider: %v", err)
 		}
-		if fi.StringValue(descResp.Url) == fi.StringValue(e.URL) {
+		// AWS does not return the https:// in the url
+		actualURL := aws.StringValue(descResp.Url)
+		if !strings.Contains(actualURL, "://") {
+			actualURL = "https://" + actualURL
+		}
+
+		if actualURL == fi.StringValue(e.URL) {
+			var actualThumbprints []fi.Resource
+			for _, thumbprint := range descResp.ThumbprintList {
+				s := aws.StringValue(thumbprint)
+				actualThumbprints = append(actualThumbprints, fi.NewStringResource(s))
+			}
+
 			actual := &IAMOIDCProvider{
 				ClientIDs:   descResp.ClientIDList,
-				Thumbprints: descResp.ThumbprintList,
+				Thumbprints: actualThumbprints,
-				URL:         descResp.Url,
+				URL:         &actualURL,
-				ARN:         arn,
+				arn:         arn,
 			}
+
+			actual.Lifecycle = e.Lifecycle
+			actual.Name = e.Name
+
 			klog.V(2).Infof("found matching IAMOIDCProvider %q", aws.StringValue(arn))
 			return actual, nil
 		}
@@ -83,17 +101,17 @@ func (e *IAMOIDCProvider) Run(c *fi.Context) error {
 }
 
 func (s *IAMOIDCProvider) CheckChanges(a, e, changes *IAMOIDCProvider) error {
+	if e.URL == nil {
+		return fi.RequiredField("URL")
+	}
+	if e.ClientIDs == nil {
+		return fi.RequiredField("ClientIDs")
+	}
+	if len(e.Thumbprints) == 0 {
+		return fi.RequiredField("Thumbprints")
+	}
+
 	if a != nil {
-		if e.URL == nil {
-			return fi.RequiredField("URL")
-		}
-		if e.ClientIDs == nil {
-			return fi.RequiredField("ClientIDs")
-		}
-		if len(e.Thumbprints) == 0 {
-			return fi.RequiredField("Thumbprints")
-		}
-	} else {
 		if changes.ClientIDs != nil {
 			return fi.CannotChangeField("ClientIDs")
 		}
@@ -105,13 +123,17 @@ func (s *IAMOIDCProvider) CheckChanges(a, e, changes *IAMOIDCProvider) error {
 }
 
 func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOIDCProvider) error {
-	if a == nil {
+	thumbprints, err := p.thumbprintsAsStrings()
+	if err != nil {
+		return err
+	}
 
+	if a == nil {
 		klog.V(2).Infof("Creating IAMOIDCProvider with Name:%q", *e.Name)
 
 		request := &iam.CreateOpenIDConnectProviderInput{
 			ClientIDList:   e.ClientIDs,
-			ThumbprintList: e.Thumbprints,
+			ThumbprintList: aws.StringSlice(thumbprints),
 			Url:            e.URL,
 		}
 
@@ -120,14 +142,14 @@ func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOID
 			return fmt.Errorf("error creating IAMOIDCProvider: %v", err)
 		}
 
-		e.ARN = response.OpenIDConnectProviderArn
+		e.arn = response.OpenIDConnectProviderArn
 	} else {
 		if changes.Thumbprints != nil {
-			klog.V(2).Infof("Updating IAMOIDCProvider Thumbprints %q", *e.ARN)
+			klog.V(2).Infof("Updating IAMOIDCProvider Thumbprints %q", fi.StringValue(e.arn))
 
 			request := &iam.UpdateOpenIDConnectProviderThumbprintInput{}
-			request.OpenIDConnectProviderArn = e.ARN
+			request.OpenIDConnectProviderArn = a.arn
-			request.ThumbprintList = e.Thumbprints
+			request.ThumbprintList = aws.StringSlice(thumbprints)
 
 			_, err := t.Cloud.IAM().UpdateOpenIDConnectProviderThumbprint(request)
 			if err != nil {
@@ -138,22 +160,37 @@ func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOID
 	return nil
 }
 
+func (p *IAMOIDCProvider) thumbprintsAsStrings() ([]string, error) {
+	var list []string
+	for _, thumbprint := range p.Thumbprints {
+		s, err := fi.ResourceAsString(thumbprint)
+		if err != nil {
+			return nil, fmt.Errorf("error getting resource as string: %v", err)
+		}
+
+		list = append(list, s)
+	}
+	return list, nil
+}
+
 type terraformIAMOIDCProvider struct {
 	URL            *string   `json:"url" cty:"url"`
 	ClientIDList   []*string `json:"client_id_list" cty:"client_id_list"`
 	ThumbprintList []*string `json:"thumbprint_list" cty:"thumbprint_list"`
 
-	Name             *string            `json:"name" cty:"name"`
 	AssumeRolePolicy *terraform.Literal `json:"assume_role_policy" cty:"assume_role_policy"`
 }
 
 func (p *IAMOIDCProvider) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *IAMOIDCProvider) error {
+	thumbprints, err := p.thumbprintsAsStrings()
+	if err != nil {
+		return err
+	}
 
 	tf := &terraformIAMOIDCProvider{
-		Name:           e.Name,
 		URL:            e.URL,
 		ClientIDList:   e.ClientIDs,
-		ThumbprintList: e.Thumbprints,
+		ThumbprintList: aws.StringSlice(thumbprints),
 	}
 
 	return t.RenderResource("aws_iam_openid_connect_provider", *e.Name, tf)

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -181,6 +181,25 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 		}
 	}
 
+	if featureflag.PublicJWKS.Enabled() {
+		key := "anonymous-issuer-discovery.addons.k8s.io"
+		version := "1.19.0-alpha.3"
+
+		{
+			location := key + "/k8s-1.16.yaml"
+			id := "k8s-1.16"
+
+			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				Name:              fi.String(key),
+				Version:           fi.String(version),
+				Selector:          map[string]string{"k8s-addon": key},
+				Manifest:          fi.String(location),
+				KubernetesVersion: ">=1.16.0-alpha.0",
+				Id:                id,
+			})
+		}
+	}
+
 	{
 		key := "core.addons.k8s.io"
 		version := "1.4.0"

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -24,6 +24,7 @@ import (
 	kopsapi "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/client/simple/vfsclientset"
+	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/kopscodecs"
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/model/iam"
@@ -50,6 +51,20 @@ func TestBootstrapChannelBuilder_BuildTasks(t *testing.T) {
 	runChannelBuilderTest(t, "awsiamauthenticator", []string{"authentication.aws-k8s-1.12"})
 }
 
+func TestBootstrapChannelBuilder_PublicJWKS(t *testing.T) {
+	h := testutils.NewIntegrationTestHarness(t)
+	defer h.Close()
+
+	h.SetupMockAWS()
+
+	featureflag.ParseFlags("+PublicJWKS")
+	unsetFeatureFlag := func() {
+		featureflag.ParseFlags("-PublicJWKS")
+	}
+	defer unsetFeatureFlag()
+	runChannelBuilderTest(t, "public-jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-issuer-discovery.addons.k8s.io-k8s-1.16"})
+}
+
 func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) {
 	basedir := path.Join("tests/bootstrapchannelbuilder/", key)
 

upup/pkg/fi/cloudup/populate_cluster_spec.go

@@ -283,6 +283,7 @@ func (c *populateClusterSpec) run(clientset simple.Clientset) error {
 			codeModels = append(codeModels, &components.KubeProxyOptionsBuilder{Context: optionsContext})
 			codeModels = append(codeModels, &components.CiliumOptionsBuilder{Context: optionsContext})
 			codeModels = append(codeModels, &components.OpenStackOptionsBulder{Context: optionsContext})
+			codeModels = append(codeModels, &components.DiscoveryOptionsBuilder{OptionsContext: optionsContext})
 		}
 	}
 

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/anonymous-issuer-discovery.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: anonymous-access.addons.k8s.io
+  name: anonymous:service-account-issuer-discovery
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:service-account-issuer-discovery
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:anonymous

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/cluster.yaml

@@ -0,0 +1,42 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  addons:
+    - manifest: s3://somebucket/example.yaml
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: events
+  iam: {}
+  kubernetesVersion: v1.14.6
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  additionalSans:
+  - proxy.api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml

@@ -0,0 +1,109 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-addon: dns-controller.addons.k8s.io
+    k8s-app: dns-controller
+    version: v1.19.0-alpha.3
+  name: dns-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: dns-controller
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-addon: dns-controller.addons.k8s.io
+        k8s-app: dns-controller
+        version: v1.19.0-alpha.3
+    spec:
+      containers:
+      - command:
+        - /dns-controller
+        - --watch-ingress=false
+        - --dns=aws-route53
+        - --zone=*/Z1AFAKE1ZON3YO
+        - --zone=*/*
+        - -v=2
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: k8s.gcr.io/kops/dns-controller:1.19.0-alpha.3
+        name: dns-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        securityContext:
+          runAsNonRoot: true
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      priorityClassName: system-cluster-critical
+      serviceAccount: dns-controller
+      tolerations:
+      - operator: Exists
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: dns-controller.addons.k8s.io
+  name: dns-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: dns-controller.addons.k8s.io
+  name: kops:dns-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - ingress
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: dns-controller.addons.k8s.io
+  name: kops:dns-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops:dns-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:dns-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,175 @@
+apiVersion: v1
+data:
+  config.yaml: |
+    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"}
+kind: ConfigMap
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.19.0-alpha.3
+  name: kops-controller
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.19.0-alpha.3
+    spec:
+      containers:
+      - command:
+        - /kops-controller
+        - --v=2
+        - --conf=/etc/kubernetes/kops-controller/config/config.yaml
+        image: k8s.gcr.io/kops/kops-controller:1.19.0-alpha.3
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        securityContext:
+          runAsNonRoot: true
+        volumeMounts:
+        - mountPath: /etc/kubernetes/kops-controller/config/
+          name: kops-controller-config
+        - mountPath: /etc/kubernetes/kops-controller/pki/
+          name: kops-controller-pki
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      priorityClassName: system-node-critical
+      serviceAccount: kops-controller
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      volumes:
+      - configMap:
+          name: kops-controller
+        name: kops-controller-config
+      - hostPath:
+          path: /etc/kubernetes/kops-controller/
+          type: Directory
+        name: kops-controller-pki
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - kops-controller-leader
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml

@@ -0,0 +1,96 @@
+kind: Addons
+metadata:
+  creationTimestamp: null
+  name: bootstrap
+spec:
+  addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: 892363f3724b8af9a564d5ef3efd010418bad7b7
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.19.0-alpha.3
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: d01bb2f3c12819e21bf0197624b95fb53dc0951a
+    name: anonymous-issuer-discovery.addons.k8s.io
+    selector:
+      k8s-addon: anonymous-issuer-discovery.addons.k8s.io
+    version: 1.19.0-alpha.3
+  - manifest: core.addons.k8s.io/v1.4.0.yaml
+    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    name: core.addons.k8s.io
+    selector:
+      k8s-addon: core.addons.k8s.io
+    version: 1.4.0
+  - id: k8s-1.6
+    kubernetesVersion: <1.12.0
+    manifest: kube-dns.addons.k8s.io/k8s-1.6.yaml
+    manifestHash: a50e6a4c2f800b4af4ac0d80edf7762cfc1de9e3
+    name: kube-dns.addons.k8s.io
+    selector:
+      k8s-addon: kube-dns.addons.k8s.io
+    version: 1.15.13-kops.3
+  - id: k8s-1.12
+    kubernetesVersion: '>=1.12.0'
+    manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    name: kube-dns.addons.k8s.io
+    selector:
+      k8s-addon: kube-dns.addons.k8s.io
+    version: 1.15.13-kops.3
+  - id: k8s-1.8
+    manifest: rbac.addons.k8s.io/k8s-1.8.yaml
+    manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914
+    name: rbac.addons.k8s.io
+    selector:
+      k8s-addon: rbac.addons.k8s.io
+    version: 1.8.0
+  - id: k8s-1.9
+    manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
+    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    name: kubelet-api.rbac.addons.k8s.io
+    selector:
+      k8s-addon: kubelet-api.rbac.addons.k8s.io
+    version: v0.0.1
+  - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
+    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    name: limit-range.addons.k8s.io
+    selector:
+      k8s-addon: limit-range.addons.k8s.io
+    version: 1.5.0
+  - id: k8s-1.6
+    kubernetesVersion: <1.12.0
+    manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml
+    manifestHash: 2a12a3eee8ea83e43106b6f7d7ea978a14e9b246
+    name: dns-controller.addons.k8s.io
+    selector:
+      k8s-addon: dns-controller.addons.k8s.io
+    version: 1.19.0-alpha.3
+  - id: k8s-1.12
+    kubernetesVersion: '>=1.12.0'
+    manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: 916cfc0d1bd7d7c6c80e85045d768c4de9178c62
+    name: dns-controller.addons.k8s.io
+    selector:
+      k8s-addon: dns-controller.addons.k8s.io
+    version: 1.19.0-alpha.3
+  - id: v1.15.0
+    kubernetesVersion: '>=1.15.0'
+    manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
+    manifestHash: 00cf6e46e25b736b2da93c6025ce482474d83904
+    name: storage-aws.addons.k8s.io
+    selector:
+      k8s-addon: storage-aws.addons.k8s.io
+    version: 1.15.0
+  - id: v1.7.0
+    kubernetesVersion: <1.15.0
+    manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
+    manifestHash: 62705a596142e6cc283280e8aa973e51536994c5
+    name: storage-aws.addons.k8s.io
+    selector:
+      k8s-addon: storage-aws.addons.k8s.io
+    version: 1.15.0

upup/pkg/fi/fitasks/keypair.go

@@ -17,6 +17,7 @@ limitations under the License.
 package fitasks
 
 import (
+	"crypto/sha1"
 	"crypto/x509/pkix"
 	"fmt"
 	"sort"
@@ -44,6 +45,9 @@ type Keypair struct {
 	Type string `json:"type"`
 	// LegacyFormat is whether the keypair is stored in a legacy format.
 	LegacyFormat bool `json:"oldFormat"`
+
+	certificate                *fi.TaskDependentResource
+	certificateSHA1Fingerprint *fi.TaskDependentResource
 }
 
 var _ fi.HasCheckExisting = &Keypair{}
@@ -98,6 +102,10 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
 	// Avoid spurious changes
 	actual.Lifecycle = e.Lifecycle
 
+	if err := e.setResources(cert); err != nil {
+		return nil, fmt.Errorf("error setting resources: %v", err)
+	}
+
 	return actual, nil
 }
 
@@ -268,3 +276,33 @@ func parsePkixName(s string) (*pkix.Name, error) {
 
 	return name, nil
 }
+
+func (e *Keypair) ensureResources() {
+	if e.certificate == nil {
+		e.certificate = &fi.TaskDependentResource{Task: e}
+	}
+	if e.certificateSHA1Fingerprint == nil {
+		e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e}
+	}
+}
+
+func (e *Keypair) setResources(cert *pki.Certificate) error {
+	e.ensureResources()
+
+	s, err := cert.AsString()
+	if err != nil {
+		return err
+	}
+	e.certificate.Resource = fi.NewStringResource(s)
+
+	fingerprint := sha1.Sum(cert.Certificate.Raw)
+	hex := fmt.Sprintf("%x", fingerprint)
+	e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex)
+
+	return nil
+}
+
+func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource {
+	e.ensureResources()
+	return e.certificateSHA1Fingerprint
+}

upup/pkg/fi/topological_sort.go

@@ -111,7 +111,7 @@ func getDependencies(tasks map[string]Task, v reflect.Value) []Task {
 			} else if dep, ok := intf.(Task); ok {
 				dependencies = append(dependencies, dep)
 			} else if _, ok := intf.(Resource); ok {
-				// Ignore: not a dependency (?)
+				// Ignore: not a dependency, unless we explicitly implement HasDependencies (e.g. TaskDependentResource)
 			} else if _, ok := intf.(*ResourceHolder); ok {
 				// Ignore: not a dependency (?)
 			} else {