Merge pull request #11016 from olemarkus/irsa-custom

user-configurable IAM roles for ServiceAccounts

cmd/kops/integration_test.go

@@ -419,6 +419,14 @@ func TestNTHQueueProcessor(t *testing.T) {
 	newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").runTestCloudformation(t)
 }
 
+// TestCustomIRSA runs a simple configuration, but with some additional IAM roles for ServiceAccounts
+func TestCustomIRSA(t *testing.T) {
+	newIntegrationTest("minimal.example.com", "irsa").
+		withServiceAccountRole("myserviceaccount.default", false).
+		withServiceAccountRole("myotherserviceaccount.myapp", true).
+		runTestTerraformAWS(t)
+}
+
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
 	ctx := context.Background()
 
@@ -620,7 +628,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 			}...)
 		}
 	}
-
 	expectedFilenames = append(expectedFilenames, i.expectServiceAccountRolePolicies...)
 
 	i.runTest(t, h, expectedFilenames, tfFileName, tfFileName, nil)

k8s/crds/kops.k8s.io_clusters.yaml

@@ -1163,6 +1163,39 @@ spec:
                     type: boolean
                   permissionsBoundary:
                     type: string
+                  serviceAccountExternalPermissions:
+                    description: ServiceAccountExternalPermissions defines the relatinship
+                      between Kubernetes ServiceAccounts and permissions with external
+                      resources.
+                    items:
+                      description: ServiceAccountExternalPermissions grants a ServiceAccount
+                        permissions to external resources.
+                      properties:
+                        aws:
+                          description: AWS grants permissions to AWS resources.
+                          properties:
+                            inlinePolicy:
+                              description: InlinePolicy is an IAM Policy that will
+                                be attached inline to the IAM Role.
+                              type: string
+                            policyARNs:
+                              description: PolicyARNs is a list of existing IAM Policies.
+                              items:
+                                type: string
+                              type: array
+                          type: object
+                        name:
+                          description: Name is the name of the Kubernetes ServiceAccount.
+                          type: string
+                        namespace:
+                          description: Namespace is the namespace of the Kubernetes
+                            ServiceAccount.
+                          type: string
+                      required:
+                      - name
+                      - namespace
+                      type: object
+                    type: array
                 required:
                 - legacy
                 type: object

pkg/apis/kops/cluster.go

@@ -218,6 +218,24 @@ type ServiceAccountIssuerDiscoveryConfig struct {
 	EnableAWSOIDCProvider bool `json:"enableAWSOIDCProvider,omitempty"`
 }
 
+// ServiceAccountExternalPermissions grants a ServiceAccount permissions to external resources.
+type ServiceAccountExternalPermission struct {
+	// Name is the name of the Kubernetes ServiceAccount.
+	Name string `json:"name"`
+	// Namespace is the namespace of the Kubernetes ServiceAccount.
+	Namespace string `json:"namespace"`
+	// AWS grants permissions to AWS resources.
+	AWS *AWSPermission `json:"aws,omitempty"`
+}
+
+// AWSPermission grants permissions to AWS resources.
+type AWSPermission struct {
+	// PolicyARNs is a list of existing IAM Policies.
+	PolicyARNs []string `json:"policyARNs,omitempty"`
+	// InlinePolicy is an IAM Policy that will be attached inline to the IAM Role.
+	InlinePolicy string `json:"inlinePolicy,omitempty"`
+}
+
 // NodeAuthorizationSpec is used to node authorization
 type NodeAuthorizationSpec struct {
 	// NodeAuthorizer defined the configuration for the node authorizer
@@ -280,6 +298,8 @@ type IAMSpec struct {
 	Legacy                 bool    `json:"legacy"`
 	AllowContainerRegistry bool    `json:"allowContainerRegistry,omitempty"`
 	PermissionsBoundary    *string `json:"permissionsBoundary,omitempty"`
+	// ServiceAccountExternalPermissions defines the relatinship between Kubernetes ServiceAccounts and permissions with external resources.
+	ServiceAccountExternalPermissions []ServiceAccountExternalPermission `json:"serviceAccountExternalPermissions,omitempty"`
 }
 
 // HookSpec is a definition hook

pkg/apis/kops/v1alpha2/cluster.go

@@ -217,6 +217,24 @@ type ServiceAccountIssuerDiscoveryConfig struct {
 	EnableAWSOIDCProvider bool `json:"enableAWSOIDCProvider,omitempty"`
 }
 
+// ServiceAccountExternalPermissions grants a ServiceAccount permissions to external resources.
+type ServiceAccountExternalPermission struct {
+	// Name is the name of the Kubernetes ServiceAccount.
+	Name string `json:"name"`
+	// Namespace is the namespace of the Kubernetes ServiceAccount.
+	Namespace string `json:"namespace"`
+	// AWS grants permissions to AWS resources.
+	AWS *AWSPermission `json:"aws,omitempty"`
+}
+
+// AWSPermission grants permissions to AWS resources.
+type AWSPermission struct {
+	// PolicyARNs is a list of existing IAM Policies.
+	PolicyARNs []string `json:"policyARNs,omitempty"`
+	// InlinePolicy is an IAM Policy that will be attached inline to the IAM Role.
+	InlinePolicy string `json:"inlinePolicy,omitempty"`
+}
+
 // NodeAuthorizationSpec is used to node authorization
 type NodeAuthorizationSpec struct {
 	// NodeAuthorizer defined the configuration for the node authorizer
@@ -278,6 +296,8 @@ type IAMSpec struct {
 	Legacy                 bool    `json:"legacy"`
 	AllowContainerRegistry bool    `json:"allowContainerRegistry,omitempty"`
 	PermissionsBoundary    *string `json:"permissionsBoundary,omitempty"`
+	// ServiceAccountExternalPermissions defines the relatinship between Kubernetes ServiceAccounts and permissions with external resources.
+	ServiceAccountExternalPermissions []ServiceAccountExternalPermission `json:"serviceAccountExternalPermissions,omitempty"`
 }
 
 // HookSpec is a definition hook

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -53,6 +53,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*AWSPermission)(nil), (*kops.AWSPermission)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(a.(*AWSPermission), b.(*kops.AWSPermission), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.AWSPermission)(nil), (*AWSPermission)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(a.(*kops.AWSPermission), b.(*AWSPermission), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*AccessSpec)(nil), (*kops.AccessSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_AccessSpec_To_kops_AccessSpec(a.(*AccessSpec), b.(*kops.AccessSpec), scope)
 	}); err != nil {
@@ -983,6 +993,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*ServiceAccountExternalPermission)(nil), (*kops.ServiceAccountExternalPermission)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(a.(*ServiceAccountExternalPermission), b.(*kops.ServiceAccountExternalPermission), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.ServiceAccountExternalPermission)(nil), (*ServiceAccountExternalPermission)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(a.(*kops.ServiceAccountExternalPermission), b.(*ServiceAccountExternalPermission), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*ServiceAccountIssuerDiscoveryConfig)(nil), (*kops.ServiceAccountIssuerDiscoveryConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_ServiceAccountIssuerDiscoveryConfig_To_kops_ServiceAccountIssuerDiscoveryConfig(a.(*ServiceAccountIssuerDiscoveryConfig), b.(*kops.ServiceAccountIssuerDiscoveryConfig), scope)
 	}); err != nil {
@@ -1122,6 +1142,28 @@ func Convert_kops_AWSLoadBalancerControllerConfig_To_v1alpha2_AWSLoadBalancerCon
 	return autoConvert_kops_AWSLoadBalancerControllerConfig_To_v1alpha2_AWSLoadBalancerControllerConfig(in, out, s)
 }
 
+func autoConvert_v1alpha2_AWSPermission_To_kops_AWSPermission(in *AWSPermission, out *kops.AWSPermission, s conversion.Scope) error {
+	out.PolicyARNs = in.PolicyARNs
+	out.InlinePolicy = in.InlinePolicy
+	return nil
+}
+
+// Convert_v1alpha2_AWSPermission_To_kops_AWSPermission is an autogenerated conversion function.
+func Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(in *AWSPermission, out *kops.AWSPermission, s conversion.Scope) error {
+	return autoConvert_v1alpha2_AWSPermission_To_kops_AWSPermission(in, out, s)
+}
+
+func autoConvert_kops_AWSPermission_To_v1alpha2_AWSPermission(in *kops.AWSPermission, out *AWSPermission, s conversion.Scope) error {
+	out.PolicyARNs = in.PolicyARNs
+	out.InlinePolicy = in.InlinePolicy
+	return nil
+}
+
+// Convert_kops_AWSPermission_To_v1alpha2_AWSPermission is an autogenerated conversion function.
+func Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(in *kops.AWSPermission, out *AWSPermission, s conversion.Scope) error {
+	return autoConvert_kops_AWSPermission_To_v1alpha2_AWSPermission(in, out, s)
+}
+
 func autoConvert_v1alpha2_AccessSpec_To_kops_AccessSpec(in *AccessSpec, out *kops.AccessSpec, s conversion.Scope) error {
 	if in.DNS != nil {
 		in, out := &in.DNS, &out.DNS
@@ -3822,6 +3864,17 @@ func autoConvert_v1alpha2_IAMSpec_To_kops_IAMSpec(in *IAMSpec, out *kops.IAMSpec
 	out.Legacy = in.Legacy
 	out.AllowContainerRegistry = in.AllowContainerRegistry
 	out.PermissionsBoundary = in.PermissionsBoundary
+	if in.ServiceAccountExternalPermissions != nil {
+		in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions
+		*out = make([]kops.ServiceAccountExternalPermission, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ServiceAccountExternalPermissions = nil
+	}
 	return nil
 }
 
@@ -3834,6 +3887,17 @@ func autoConvert_kops_IAMSpec_To_v1alpha2_IAMSpec(in *kops.IAMSpec, out *IAMSpec
 	out.Legacy = in.Legacy
 	out.AllowContainerRegistry = in.AllowContainerRegistry
 	out.PermissionsBoundary = in.PermissionsBoundary
+	if in.ServiceAccountExternalPermissions != nil {
+		in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions
+		*out = make([]ServiceAccountExternalPermission, len(*in))
+		for i := range *in {
+			if err := Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ServiceAccountExternalPermissions = nil
+	}
 	return nil
 }
 
@@ -6266,6 +6330,46 @@ func Convert_kops_SSHCredentialSpec_To_v1alpha2_SSHCredentialSpec(in *kops.SSHCr
 	return autoConvert_kops_SSHCredentialSpec_To_v1alpha2_SSHCredentialSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in *ServiceAccountExternalPermission, out *kops.ServiceAccountExternalPermission, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Namespace = in.Namespace
+	if in.AWS != nil {
+		in, out := &in.AWS, &out.AWS
+		*out = new(kops.AWSPermission)
+		if err := Convert_v1alpha2_AWSPermission_To_kops_AWSPermission(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AWS = nil
+	}
+	return nil
+}
+
+// Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission is an autogenerated conversion function.
+func Convert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in *ServiceAccountExternalPermission, out *kops.ServiceAccountExternalPermission, s conversion.Scope) error {
+	return autoConvert_v1alpha2_ServiceAccountExternalPermission_To_kops_ServiceAccountExternalPermission(in, out, s)
+}
+
+func autoConvert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in *kops.ServiceAccountExternalPermission, out *ServiceAccountExternalPermission, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Namespace = in.Namespace
+	if in.AWS != nil {
+		in, out := &in.AWS, &out.AWS
+		*out = new(AWSPermission)
+		if err := Convert_kops_AWSPermission_To_v1alpha2_AWSPermission(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.AWS = nil
+	}
+	return nil
+}
+
+// Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission is an autogenerated conversion function.
+func Convert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in *kops.ServiceAccountExternalPermission, out *ServiceAccountExternalPermission, s conversion.Scope) error {
+	return autoConvert_kops_ServiceAccountExternalPermission_To_v1alpha2_ServiceAccountExternalPermission(in, out, s)
+}
+
 func autoConvert_v1alpha2_ServiceAccountIssuerDiscoveryConfig_To_kops_ServiceAccountIssuerDiscoveryConfig(in *ServiceAccountIssuerDiscoveryConfig, out *kops.ServiceAccountIssuerDiscoveryConfig, s conversion.Scope) error {
 	out.DiscoveryStore = in.DiscoveryStore
 	out.EnableAWSOIDCProvider = in.EnableAWSOIDCProvider

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -83,6 +83,27 @@ func (in *AWSLoadBalancerControllerConfig) DeepCopy() *AWSLoadBalancerController
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AWSPermission) DeepCopyInto(out *AWSPermission) {
+	*out = *in
+	if in.PolicyARNs != nil {
+		in, out := &in.PolicyARNs, &out.PolicyARNs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSPermission.
+func (in *AWSPermission) DeepCopy() *AWSPermission {
+	if in == nil {
+		return nil
+	}
+	out := new(AWSPermission)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AccessSpec) DeepCopyInto(out *AccessSpec) {
 	*out = *in
@@ -1944,6 +1965,13 @@ func (in *IAMSpec) DeepCopyInto(out *IAMSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.ServiceAccountExternalPermissions != nil {
+		in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions
+		*out = make([]ServiceAccountExternalPermission, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
@@ -4375,6 +4403,27 @@ func (in *SSHCredentialSpec) DeepCopy() *SSHCredentialSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ServiceAccountExternalPermission) DeepCopyInto(out *ServiceAccountExternalPermission) {
+	*out = *in
+	if in.AWS != nil {
+		in, out := &in.AWS, &out.AWS
+		*out = new(AWSPermission)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountExternalPermission.
+func (in *ServiceAccountExternalPermission) DeepCopy() *ServiceAccountExternalPermission {
+	if in == nil {
+		return nil
+	}
+	out := new(ServiceAccountExternalPermission)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceAccountIssuerDiscoveryConfig) DeepCopyInto(out *ServiceAccountIssuerDiscoveryConfig) {
 	*out = *in

pkg/apis/kops/validation/validation.go

@@ -254,6 +254,52 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		}
 	}
 
+	if spec.IAM != nil {
+		if len(spec.IAM.ServiceAccountExternalPermissions) > 0 {
+			if spec.ServiceAccountIssuerDiscovery == nil || !spec.ServiceAccountIssuerDiscovery.EnableAWSOIDCProvider {
+				allErrs = append(allErrs, field.Forbidden(fieldPath.Child("iam", "serviceAccountExternalPermissions"), "serviceAccountExternalPermissions requires AWS OIDC Provider to be enabled"))
+			}
+			allErrs = append(allErrs, validateSAExternalPermissions(spec.IAM.ServiceAccountExternalPermissions, fieldPath.Child("iam", "serviceAccountExternalPermissions"))...)
+		}
+	}
+
+	return allErrs
+}
+
+func validateSAExternalPermissions(externalPermissions []kops.ServiceAccountExternalPermission, path *field.Path) (allErrs field.ErrorList) {
+	if len(externalPermissions) == 0 {
+		return allErrs
+	}
+
+	sas := make(map[string]string)
+	for _, sa := range externalPermissions {
+		key := fmt.Sprintf("%s/%s", sa.Namespace, sa.Name)
+		p := path.Key(key)
+		if sa.Namespace == "" {
+			allErrs = append(allErrs, field.Required(p.Child("namespace"), "namespace cannot be empty"))
+		}
+		if sa.Name == "" {
+			allErrs = append(allErrs, field.Required(p.Child("name"), "name cannot be empty"))
+		}
+		_, duplicate := sas[key]
+		if duplicate {
+			allErrs = append(allErrs, field.Duplicate(p, key))
+		}
+		sas[key] = ""
+		aws := sa.AWS
+		ap := p.Child("aws")
+		if aws == nil {
+			allErrs = append(allErrs, field.Required(ap, "AWS permissions must be set"))
+			continue
+		}
+
+		if len(aws.PolicyARNs) == 0 && aws.InlinePolicy == "" {
+			allErrs = append(allErrs, field.Required(ap, "either inlinePolicy or policyARN must be set"))
+		}
+		if len(aws.PolicyARNs) > 0 && aws.InlinePolicy != "" {
+			allErrs = append(allErrs, field.Forbidden(ap, "cannot set both inlinePolicy and policyARN"))
+		}
+	}
 	return allErrs
 }
 

pkg/apis/kops/validation/validation_test.go

@@ -1191,3 +1191,92 @@ func Test_Validate_CloudConfiguration(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateSAExternalPermissions(t *testing.T) {
+	grid := []struct {
+		Description    string
+		Input          []kops.ServiceAccountExternalPermission
+		ExpectedErrors []string
+	}{
+
+		{
+			Description: "Duplicate SA",
+			Input: []kops.ServiceAccountExternalPermission{
+				{
+					Name:      "MySA",
+					Namespace: "MyNS",
+					AWS: &kops.AWSPermission{
+						PolicyARNs: []string{"-"},
+					},
+				},
+				{
+					Name:      "MySA",
+					Namespace: "MyNS",
+					AWS: &kops.AWSPermission{
+						PolicyARNs: []string{"-"},
+					},
+				},
+			},
+			ExpectedErrors: []string{"Duplicate value::iam.serviceAccountExternalPermissions[MyNS/MySA]"},
+		},
+		{
+			Description: "Missing permissions",
+			Input: []kops.ServiceAccountExternalPermission{
+				{
+					Name:      "MySA",
+					Namespace: "MyNS",
+				},
+			},
+			ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[MyNS/MySA].aws"},
+		},
+		{
+			Description: "Setting both arn and inline",
+			Input: []kops.ServiceAccountExternalPermission{
+				{
+					Name:      "MySA",
+					Namespace: "MyNS",
+					AWS: &kops.AWSPermission{
+						PolicyARNs:   []string{"-"},
+						InlinePolicy: "-",
+					},
+				},
+			},
+			ExpectedErrors: []string{"Forbidden::iam.serviceAccountExternalPermissions[MyNS/MySA].aws"},
+		},
+		{
+			Description: "Empty SA name",
+			Input: []kops.ServiceAccountExternalPermission{
+				{
+					Namespace: "MyNS",
+					AWS: &kops.AWSPermission{
+						PolicyARNs:   []string{"-"},
+						InlinePolicy: "-",
+					},
+				},
+			},
+			ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[MyNS/].name"},
+		},
+		{
+			Description: "Empty SA namespace",
+			Input: []kops.ServiceAccountExternalPermission{
+				{
+					Name: "MySA",
+					AWS: &kops.AWSPermission{
+						PolicyARNs:   []string{"-"},
+						InlinePolicy: "-",
+					},
+				},
+			},
+			ExpectedErrors: []string{"Required value::iam.serviceAccountExternalPermissions[/MySA].namespace"},
+		},
+	}
+
+	for _, g := range grid {
+		fldPath := field.NewPath("iam.serviceAccountExternalPermissions")
+		t.Run(g.Description, func(t *testing.T) {
+			errs := validateSAExternalPermissions(g.Input, fldPath)
+			testErrors(t, g.Input, errs, g.ExpectedErrors)
+		})
+	}
+
+}

pkg/apis/kops/zz_generated.deepcopy.go

@@ -83,6 +83,27 @@ func (in *AWSLoadBalancerControllerConfig) DeepCopy() *AWSLoadBalancerController
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AWSPermission) DeepCopyInto(out *AWSPermission) {
+	*out = *in
+	if in.PolicyARNs != nil {
+		in, out := &in.PolicyARNs, &out.PolicyARNs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSPermission.
+func (in *AWSPermission) DeepCopy() *AWSPermission {
+	if in == nil {
+		return nil
+	}
+	out := new(AWSPermission)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AccessSpec) DeepCopyInto(out *AccessSpec) {
 	*out = *in
@@ -2110,6 +2131,13 @@ func (in *IAMSpec) DeepCopyInto(out *IAMSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.ServiceAccountExternalPermissions != nil {
+		in, out := &in.ServiceAccountExternalPermissions, &out.ServiceAccountExternalPermissions
+		*out = make([]ServiceAccountExternalPermission, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
@@ -4589,6 +4617,27 @@ func (in *SSHCredentialSpec) DeepCopy() *SSHCredentialSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ServiceAccountExternalPermission) DeepCopyInto(out *ServiceAccountExternalPermission) {
+	*out = *in
+	if in.AWS != nil {
+		in, out := &in.AWS, &out.AWS
+		*out = new(AWSPermission)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountExternalPermission.
+func (in *ServiceAccountExternalPermission) DeepCopy() *ServiceAccountExternalPermission {
+	if in == nil {
+		return nil
+	}
+	out := new(ServiceAccountExternalPermission)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceAccountIssuerDiscoveryConfig) DeepCopyInto(out *ServiceAccountIssuerDiscoveryConfig) {
 	*out = *in

pkg/featureflag/featureflag.go

@@ -91,9 +91,9 @@ var (
 	LegacyIAM = New("LegacyIAM", Bool(false))
 	// ClusterAddons activates experimental cluster-addons support
 	ClusterAddons = New("ClusterAddons", Bool(false))
-	// UseServiceAccountIAM controls whether we use pod-level IAM permissions for our system pods.
+	// UseServiceAccountIAM controls whether we use pod-level IAM permissions for our system pods and kOps addons.
 	UseServiceAccountIAM = New("UseServiceAccountIAM", Bool(false))
-	// PublicJWKS enables public jwks access.  This is generally not as secure as republishing.
+	// PublicJWKS enables public jwks access.
 	PublicJWKS = New("PublicJWKS", Bool(false))
 	// Azure toggles the Azure support.
 	Azure = New("Azure", Bool(false))

pkg/model/awsmodel/BUILD.bazel

@@ -35,6 +35,7 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/aws/endpoints:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/k8s.io/legacy-cloud-providers/aws:go_default_library",

pkg/model/awsmodel/iam.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/dns"
@@ -124,26 +125,64 @@ func (b *IAMModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
+	iamSpec := b.Cluster.Spec.IAM
+	if iamSpec != nil {
+		for _, sa := range iamSpec.ServiceAccountExternalPermissions {
+			var p *iam.Policy
+			aws := sa.AWS
+			if aws.InlinePolicy != "" {
+				bp, err := b.buildPolicy(aws.InlinePolicy)
+				p = bp
+				if err != nil {
+					return fmt.Errorf("error inline policy: %w", err)
+				}
+			}
+			serviceAccount := &iam.GenericServiceAccount{
+				NamespacedName: types.NamespacedName{
+					Name:      sa.Name,
+					Namespace: sa.Namespace,
+				},
+				Policy: p,
+			}
+			iamRole, err := b.BuildServiceAccountRoleTasks(serviceAccount, c)
+			if err != nil {
+				return fmt.Errorf("error building service account role tasks: %w", err)
+			}
+			if len(aws.PolicyARNs) > 0 {
+				name := "external-" + fi.StringValue(iamRole.Name)
+				externalPolicies := aws.PolicyARNs
+				c.AddTask(&awstasks.IAMRolePolicy{
+					Name:             fi.String(name),
+					ExternalPolicies: &externalPolicies,
+					Managed:          true,
+					Role:             iamRole,
+					Lifecycle:        b.Lifecycle,
+				})
+			}
+		}
+
+	}
+
 	return nil
 }
 
 // BuildServiceAccountRoleTasks build tasks specifically for the ServiceAccount role.
-func (b *IAMModelBuilder) BuildServiceAccountRoleTasks(role iam.Subject, c *fi.ModelBuilderContext) error {
+func (b *IAMModelBuilder) BuildServiceAccountRoleTasks(role iam.Subject, c *fi.ModelBuilderContext) (*awstasks.IAMRole, error) {
 	iamName, err := b.IAMNameForServiceAccountRole(role)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	iamRole, err := b.buildIAMRole(role, iamName, c)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if err := b.buildIAMRolePolicy(role, iamName, iamRole, c); err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return iamRole, nil
 }
 
 func (b *IAMModelBuilder) buildIAMRole(role iam.Subject, iamName string, c *fi.ModelBuilderContext) (*awstasks.IAMRole, error) {
@@ -312,20 +351,14 @@ func (b *IAMModelBuilder) buildIAMTasks(role iam.Subject, iamName string, c *fi.
 			}
 
 			if additionalPolicy != "" {
-				p := &iam.Policy{
+				p, err := b.buildPolicy(additionalPolicy)
-					Version: iam.PolicyDefaultVersion,
-				}
-
-				statements, err := iam.ParseStatements(additionalPolicy)
 				if err != nil {
 					return fmt.Errorf("additionalPolicy %q is invalid: %v", roleKey, err)
 				}
 
-				p.Statement = append(p.Statement, statements...)
-
 				policy, err := p.AsJSON()
 				if err != nil {
-					return fmt.Errorf("error building IAM policy: %v", err)
+					return fmt.Errorf("error building IAM policy: %w", err)
 				}
 
 				t.PolicyDocument = fi.NewStringResource(policy)
@@ -340,6 +373,20 @@ func (b *IAMModelBuilder) buildIAMTasks(role iam.Subject, iamName string, c *fi.
 	return nil
 }
 
+func (b *IAMModelBuilder) buildPolicy(policyString string) (*iam.Policy, error) {
+	p := &iam.Policy{
+		Version: iam.PolicyDefaultVersion,
+	}
+
+	statements, err := iam.ParseStatements(policyString)
+	if err != nil {
+		return nil, err
+	}
+
+	p.Statement = append(p.Statement, statements...)
+	return p, nil
+}
+
 // IAMServiceEC2 returns the name of the IAM service for EC2 in the current region.
 // It is ec2.amazonaws.com in the default aws partition, but different in other isolated/custom partitions
 func IAMServiceEC2(region string) string {

pkg/model/iam/BUILD.bazel

@@ -33,8 +33,11 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/kops:go_default_library",
+        "//pkg/testutils:go_default_library",
         "//pkg/testutils/golden:go_default_library",
         "//pkg/util/stringorslice:go_default_library",
+        "//upup/pkg/fi:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
     ],
 )

pkg/model/iam/iam_builder.go

@@ -678,6 +678,9 @@ func (b *PolicyResource) Open() (io.Reader, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error building IAM policy: %v", err)
 	}
+	if policy == nil {
+		return bytes.NewReader([]byte{}), nil
+	}
 	j, err := policy.AsJSON()
 	if err != nil {
 		return nil, fmt.Errorf("error building IAM policy: %v", err)

pkg/model/iam/iam_builder_test.go

@@ -22,9 +22,12 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/testutils"
 	"k8s.io/kops/pkg/testutils/golden"
 	"k8s.io/kops/pkg/util/stringorslice"
+	"k8s.io/kops/upup/pkg/fi"
 )
 
 func TestIAMPrefix(t *testing.T) {
@@ -226,3 +229,34 @@ func TestPolicyGeneration(t *testing.T) {
 		golden.AssertMatchesFile(t, actualPolicy, x.Policy)
 	}
 }
+
+func TestEmptyPolicy(t *testing.T) {
+
+	role := &GenericServiceAccount{
+		NamespacedName: types.NamespacedName{
+			Name:      "myaccount",
+			Namespace: "default",
+		},
+		Policy: nil,
+	}
+
+	cluster := testutils.BuildMinimalCluster("irsa.example.com")
+	b := &PolicyBuilder{
+		Cluster: cluster,
+		Role:    role,
+	}
+
+	pr := &PolicyResource{
+		Builder: b,
+	}
+
+	policy, err := fi.ResourceAsString(pr)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if policy != "" {
+		t.Errorf("empty policy should result in empty string, but was %q", policy)
+	}
+
+}

pkg/model/iam/subject.go

@@ -77,6 +77,19 @@ func (_ *NodeRoleBastion) ServiceAccount() (types.NamespacedName, bool) {
 	return types.NamespacedName{}, false
 }
 
+type GenericServiceAccount struct {
+	NamespacedName types.NamespacedName
+	Policy         *Policy
+}
+
+func (g *GenericServiceAccount) ServiceAccount() (types.NamespacedName, bool) {
+	return g.NamespacedName, true
+}
+
+func (g *GenericServiceAccount) BuildAWSPolicy(*PolicyBuilder) (*Policy, error) {
+	return g.Policy, nil
+}
+
 // BuildNodeRoleSubject returns a Subject implementation for the specified InstanceGroupRole.
 func BuildNodeRoleSubject(igRole kops.InstanceGroupRole, enableLifecycleHookPermissions bool) (Subject, error) {
 	switch igRole {

pkg/model/issuerdiscovery.go

@@ -28,7 +28,6 @@ import (
 
 	"gopkg.in/square/go-jose.v2"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/model/iam"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
@@ -53,7 +52,8 @@ type oidcDiscovery struct {
 }
 
 func (b *IssuerDiscoveryModelBuilder) Build(c *fi.ModelBuilderContext) error {
-	if !featureflag.PublicJWKS.Enabled() {
+	said := b.Cluster.Spec.ServiceAccountIssuerDiscovery
+	if said == nil {
 		return nil
 	}
 

tests/integration/update_cluster/irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "api.internal.minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/api.internal.minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_masters.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com/oidc:sub": "system:serviceaccount:myapp:myotherserviceaccount"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com/oidc"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com/oidc:sub": "system:serviceaccount:default:myserviceaccount"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com/oidc"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_nodes.minimal.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,34 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -0,0 +1,211 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DescribeVolumesModifications",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": "autoscaling:CompleteLifecycleAction",
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": "autoscaling:DescribeLifecycleHooks",
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": "autoscaling:DescribeAutoScalingInstances",
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:CompleteLifecycleAction",
+        "autoscaling:DescribeAutoScalingInstances"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy

@@ -0,0 +1,23 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "dynamodb:*"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "es:*"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_iam_role_policy_nodes.minimal.example.com_policy

@@ -0,0 +1,22 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": "autoscaling:DescribeAutoScalingInstances",
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/irsa/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -0,0 +1,328 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+encryptionConfig: null
+etcdClusters:
+  events:
+    version: 3.4.13
+  main:
+    version: 3.4.13
+kubeAPIServer:
+  allowPrivileged: true
+  anonymousAuth: false
+  apiAudiences:
+  - kubernetes.svc.default
+  apiServerCount: 1
+  authorizationMode: AlwaysAllow
+  bindAddress: 0.0.0.0
+  cloudProvider: aws
+  enableAdmissionPlugins:
+  - NamespaceLifecycle
+  - LimitRanger
+  - ServiceAccount
+  - PersistentVolumeLabel
+  - DefaultStorageClass
+  - DefaultTolerationSeconds
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
+  - NodeRestriction
+  - ResourceQuota
+  etcdServers:
+  - http://127.0.0.1:4001
+  etcdServersOverrides:
+  - /events#http://127.0.0.1:4002
+  image: k8s.gcr.io/kube-apiserver:v1.20.0
+  kubeletPreferredAddressTypes:
+  - InternalIP
+  - Hostname
+  - ExternalIP
+  logLevel: 2
+  requestheaderAllowedNames:
+  - aggregator
+  requestheaderExtraHeaderPrefixes:
+  - X-Remote-Extra-
+  requestheaderGroupHeaders:
+  - X-Remote-Group
+  requestheaderUsernameHeaders:
+  - X-Remote-User
+  securePort: 443
+  serviceAccountIssuer: https://discovery.example.com/minimal.example.com/oidc
+  serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/oidc/openid/v1/jwks
+  serviceClusterIPRange: 100.64.0.0/13
+  storageBackend: etcd3
+kubeControllerManager:
+  allocateNodeCIDRs: true
+  attachDetachReconcileSyncPeriod: 1m0s
+  cloudProvider: aws
+  clusterCIDR: 100.96.0.0/11
+  clusterName: minimal.example.com
+  configureCloudRoutes: false
+  image: k8s.gcr.io/kube-controller-manager:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+  useServiceAccountCredentials: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubeScheduler:
+  image: k8s.gcr.io/kube-scheduler:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+masterKubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube
+  - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+  - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
+  - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
+ClusterName: minimal.example.com
+ConfigBase: memfs://clusters.example.com/minimal.example.com
+InstanceGroupName: master-us-test-1a
+InstanceGroupRole: Master
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kops.k8s.io/kops-controller-pki: ""
+    kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
+    node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+channels:
+- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+etcdManifests:
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
+staticManifests:
+- key: kube-apiserver-healthcheck
+  path: manifests/static/kube-apiserver-healthcheck.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -0,0 +1,232 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+ClusterName: minimal.example.com
+ConfigBase: memfs://clusters.example.com/minimal.example.com
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kubernetes.io/role: node
+    node-role.kubernetes.io/node: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+channels:
+- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/irsa/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/irsa/in-v1alpha2.yaml

@@ -0,0 +1,103 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  iam:
+    serviceAccountExternalPermissions:
+      - name: myserviceaccount
+        namespace: default
+        aws:
+          policyARNs:
+          - arn:aws:iam::123456789012:policy/UsersManageOwnCredentials
+      - name: myotherserviceaccount
+        namespace: myapp
+        aws:
+          inlinePolicy: |
+            [
+              {
+                "Effect": "Allow",
+                "Action": ["dynamodb:*"],
+                "Resource": ["*"]
+              },
+              {
+                "Effect": "Allow",
+                "Action": ["es:*"],
+                "Resource": ["*"]
+              }
+            ]
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.20.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    cni: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  serviceAccountIssuerDiscovery:
+    enableAWSOIDCProvider: true
+    discoveryStore: memfs://discovery.example.com/minimal.example.com
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a

tests/integration/update_cluster/irsa/kubernetes.tf

@@ -0,0 +1,706 @@
+locals {
+  cluster_name                          = "minimal.example.com"
+  default-myserviceaccount_role_arn     = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.arn
+  default-myserviceaccount_role_name    = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name
+  master_autoscaling_group_ids          = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
+  master_security_group_ids             = [aws_security_group.masters-minimal-example-com.id]
+  masters_role_arn                      = aws_iam_role.masters-minimal-example-com.arn
+  masters_role_name                     = aws_iam_role.masters-minimal-example-com.name
+  myapp-myotherserviceaccount_role_arn  = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.arn
+  myapp-myotherserviceaccount_role_name = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name
+  node_autoscaling_group_ids            = [aws_autoscaling_group.nodes-minimal-example-com.id]
+  node_security_group_ids               = [aws_security_group.nodes-minimal-example-com.id]
+  node_subnet_ids                       = [aws_subnet.us-test-1a-minimal-example-com.id]
+  nodes_role_arn                        = aws_iam_role.nodes-minimal-example-com.arn
+  nodes_role_name                       = aws_iam_role.nodes-minimal-example-com.name
+  region                                = "us-test-1"
+  route_table_public_id                 = aws_route_table.minimal-example-com.id
+  subnet_us-test-1a_id                  = aws_subnet.us-test-1a-minimal-example-com.id
+  vpc_cidr_block                        = aws_vpc.minimal-example-com.cidr_block
+  vpc_id                                = aws_vpc.minimal-example-com.id
+}
+
+output "cluster_name" {
+  value = "minimal.example.com"
+}
+
+output "default-myserviceaccount_role_arn" {
+  value = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.arn
+}
+
+output "default-myserviceaccount_role_name" {
+  value = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name
+}
+
+output "master_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
+}
+
+output "master_security_group_ids" {
+  value = [aws_security_group.masters-minimal-example-com.id]
+}
+
+output "masters_role_arn" {
+  value = aws_iam_role.masters-minimal-example-com.arn
+}
+
+output "masters_role_name" {
+  value = aws_iam_role.masters-minimal-example-com.name
+}
+
+output "myapp-myotherserviceaccount_role_arn" {
+  value = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.arn
+}
+
+output "myapp-myotherserviceaccount_role_name" {
+  value = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name
+}
+
+output "node_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.nodes-minimal-example-com.id]
+}
+
+output "node_security_group_ids" {
+  value = [aws_security_group.nodes-minimal-example-com.id]
+}
+
+output "node_subnet_ids" {
+  value = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+output "nodes_role_arn" {
+  value = aws_iam_role.nodes-minimal-example-com.arn
+}
+
+output "nodes_role_name" {
+  value = aws_iam_role.nodes-minimal-example-com.name
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "route_table_public_id" {
+  value = aws_route_table.minimal-example-com.id
+}
+
+output "subnet_us-test-1a_id" {
+  value = aws_subnet.us-test-1a-minimal-example-com.id
+}
+
+output "vpc_cidr_block" {
+  value = aws_vpc.minimal-example-com.cidr_block
+}
+
+output "vpc_id" {
+  value = aws_vpc.minimal-example-com.id
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id
+    version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version
+  }
+  max_size            = 1
+  metrics_granularity = "1Minute"
+  min_size            = 1
+  name                = "master-us-test-1a.masters.minimal.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "minimal.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "master-us-test-1a.masters.minimal.example.com"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "master"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/master"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "master-us-test-1a"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/minimal.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+resource "aws_autoscaling_group" "nodes-minimal-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.nodes-minimal-example-com.id
+    version = aws_launch_template.nodes-minimal-example-com.latest_version
+  }
+  max_size            = 2
+  metrics_granularity = "1Minute"
+  min_size            = 2
+  name                = "nodes.minimal.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "minimal.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "nodes.minimal.example.com"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "node"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/node"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "nodes"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/minimal.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.etcd-events.minimal.example.com"
+    "k8s.io/etcd/events"                        = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                        = "1"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.etcd-main.minimal.example.com"
+    "k8s.io/etcd/main"                          = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                        = "1"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_iam_instance_profile" "masters-minimal-example-com" {
+  name = "masters.minimal.example.com"
+  role = aws_iam_role.masters-minimal-example-com.name
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "masters.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_instance_profile" "nodes-minimal-example-com" {
+  name = "nodes.minimal.example.com"
+  role = aws_iam_role.nodes-minimal-example-com.name
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "nodes.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_openid_connect_provider" "minimal-example-com" {
+  client_id_list = ["amazonaws.com"]
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280", "a9d53002e97e00e043244f3d170d6f4c414104fd"]
+  url             = "https://discovery.example.com/minimal.example.com/oidc"
+}
+
+resource "aws_iam_role_policy_attachment" "external-myserviceaccount-default-sa-minimal-example-com-3186075376" {
+  policy_arn = "arn:aws:iam::123456789012:policy/UsersManageOwnCredentials"
+  role       = aws_iam_role.myserviceaccount-default-sa-minimal-example-com.name
+}
+
+resource "aws_iam_role_policy" "masters-minimal-example-com" {
+  name   = "masters.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy")
+  role   = aws_iam_role.masters-minimal-example-com.name
+}
+
+resource "aws_iam_role_policy" "myotherserviceaccount-myapp-sa-minimal-example-com" {
+  name   = "myotherserviceaccount.myapp.sa.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_myotherserviceaccount.myapp.sa.minimal.example.com_policy")
+  role   = aws_iam_role.myotherserviceaccount-myapp-sa-minimal-example-com.name
+}
+
+resource "aws_iam_role_policy" "nodes-minimal-example-com" {
+  name   = "nodes.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy")
+  role   = aws_iam_role.nodes-minimal-example-com.name
+}
+
+resource "aws_iam_role" "masters-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy")
+  name               = "masters.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "masters.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role" "myotherserviceaccount-myapp-sa-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_myotherserviceaccount.myapp.sa.minimal.example.com_policy")
+  name               = "myotherserviceaccount.myapp.sa.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "myotherserviceaccount.myapp.sa.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role" "myserviceaccount-default-sa-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_myserviceaccount.default.sa.minimal.example.com_policy")
+  name               = "myserviceaccount.default.sa.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "myserviceaccount.default.sa.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role" "nodes-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy")
+  name               = "nodes.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "nodes.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_internet_gateway" "minimal-example-com" {
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 64
+      volume_type           = "gp3"
+    }
+  }
+  block_device_mappings {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.masters-minimal-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "m3.medium"
+  key_name      = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "master-us-test-1a.masters.minimal.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.masters-minimal-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                                                     = "minimal.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/minimal.example.com"                                                             = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                                                     = "minimal.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/minimal.example.com"                                                             = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                                                     = "minimal.example.com"
+    "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+    "k8s.io/role/master"                                                                                    = "1"
+    "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+    "kubernetes.io/cluster/minimal.example.com"                                                             = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data")
+}
+
+resource "aws_launch_template" "nodes-minimal-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 128
+      volume_type           = "gp3"
+    }
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.nodes-minimal-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "t2.medium"
+  key_name      = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "nodes.minimal.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.nodes-minimal-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                          = "minimal.example.com"
+      "Name"                                                                       = "nodes.minimal.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/minimal.example.com"                                  = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                          = "minimal.example.com"
+      "Name"                                                                       = "nodes.minimal.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/minimal.example.com"                                  = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                          = "minimal.example.com"
+    "Name"                                                                       = "nodes.minimal.example.com"
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+    "k8s.io/role/node"                                                           = "1"
+    "kops.k8s.io/instancegroup"                                                  = "nodes"
+    "kubernetes.io/cluster/minimal.example.com"                                  = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data")
+}
+
+resource "aws_route_table_association" "us-test-1a-minimal-example-com" {
+  route_table_id = aws_route_table.minimal-example-com.id
+  subnet_id      = aws_subnet.us-test-1a-minimal-example-com.id
+}
+
+resource "aws_route_table" "minimal-example-com" {
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+    "kubernetes.io/kops/role"                   = "public"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_route" "route-0-0-0-0--0" {
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = aws_internet_gateway.minimal-example-com.id
+  route_table_id         = aws_route_table.minimal-example-com.id
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-minimal-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-minimal-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-minimal-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 443
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 443
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-masters-minimal-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-minimal-example-com.id
+  source_security_group_id = aws_security_group.masters-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-1to2379-masters-minimal-example-com" {
+  from_port                = 1
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 2379
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-2382to4000-masters-minimal-example-com" {
+  from_port                = 2382
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 4000
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-4003to65535-masters-minimal-example-com" {
+  from_port                = 4003
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1to65535-masters-minimal-example-com" {
+  from_port                = 1
+  protocol                 = "udp"
+  security_group_id        = aws_security_group.masters-minimal-example-com.id
+  source_security_group_id = aws_security_group.nodes-minimal-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group" "masters-minimal-example-com" {
+  description = "Security group for masters"
+  name        = "masters.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "masters.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_security_group" "nodes-minimal-example-com" {
+  description = "Security group for nodes"
+  name        = "nodes.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "nodes.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_subnet" "us-test-1a-minimal-example-com" {
+  availability_zone = "us-test-1a"
+  cidr_block        = "172.20.32.0/19"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "us-test-1a.minimal.example.com"
+    "SubnetType"                                = "Public"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+    "kubernetes.io/role/elb"                    = "1"
+  }
+  vpc_id = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_vpc_dhcp_options_association" "minimal-example-com" {
+  dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id
+  vpc_id          = aws_vpc.minimal-example-com.id
+}
+
+resource "aws_vpc_dhcp_options" "minimal-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc" "minimal-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+terraform {
+  required_version = ">= 0.12.26"
+  required_providers {
+    aws = {
+      "source"  = "hashicorp/aws"
+      "version" = ">= 3.34.0"
+    }
+  }
+}

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -431,7 +431,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			for _, serviceAccountRole := range serviceAccountRoles {
 				iamModelBuilder := &awsmodel.IAMModelBuilder{AWSModelContext: awsModelContext, Lifecycle: b.Lifecycle, Cluster: b.Cluster}
 
-				err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c)
+				_, err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c)
 				if err != nil {
 					return nil, err
 				}
@@ -593,7 +593,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			for _, serviceAccountRole := range serviceAccountRoles {
 				iamModelBuilder := &awsmodel.IAMModelBuilder{AWSModelContext: awsModelContext, Lifecycle: b.Lifecycle, Cluster: b.Cluster}
 
-				err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c)
+				_, err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c)
 				if err != nil {
 					return nil, err
 				}