diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 680999bf1b..0ac9b26838 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -52,15 +52,17 @@ import ( const updateClusterTestBase = "../../tests/integration/update_cluster/" type integrationTest struct { - clusterName string - srcDir string - version string - private bool - zones int - expectPolicies bool - launchConfiguration bool - lifecycleOverrides []string - sshKey bool + clusterName string + srcDir string + version string + private bool + zones int + expectPolicies bool + // expectServiceAccountRoles is true if we expect to assign per-ServiceAccount IAM roles (instead of just using the node roles) + expectServiceAccountRoles bool + launchConfiguration bool + lifecycleOverrides []string + sshKey bool // caKey is true if we should use a provided ca.crt & ca.key as our CA caKey bool jsonOutput bool @@ -105,6 +107,12 @@ func (i *integrationTest) withoutPolicies() *integrationTest { return i } +// withServiceAccountRoles indicates we expect to assign per-ServiceAccount IAM roles (instead of just using the node roles) +func (i *integrationTest) withServiceAccountRoles() *integrationTest { + i.expectServiceAccountRoles = true + return i +} + func (i *integrationTest) withLifecycleOverrides(lco []string) *integrationTest { i.lifecycleOverrides = lco return i @@ -332,16 +340,16 @@ func TestLaunchConfigurationASG(t *testing.T) { newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestCloudformation(t) } -// TestJWKS runs a simple configuration, but with PublicJWKS enabled +// TestPublicJWKS runs a simple configuration, but with UseServiceAccountIAM and PublicJWKS enabled func TestPublicJWKS(t *testing.T) { - featureflag.ParseFlags("+PublicJWKS") + featureflag.ParseFlags("+UseServiceAccountIAM,+PublicJWKS") unsetFeatureFlags := func() { - featureflag.ParseFlags("-PublicJWKS") + featureflag.ParseFlags("-UseServiceAccountIAM,-PublicJWKS") } defer unsetFeatureFlags() // We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration. - newIntegrationTest("minimal.example.com", "public-jwks").withCAKey().runTestTerraformAWS(t) + newIntegrationTest("minimal.example.com", "public-jwks").withCAKey().withServiceAccountRoles().runTestTerraformAWS(t) } func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) { @@ -544,6 +552,12 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { } } } + if i.expectServiceAccountRoles { + expectedFilenames = append(expectedFilenames, []string{ + "aws_iam_role_dns-controller.kube-system.sa." + i.clusterName + "_policy", + "aws_iam_role_policy_dns-controller.kube-system.sa." + i.clusterName + "_policy", + }...) + } i.runTest(t, h, expectedFilenames, tfFileName, tfFileName, nil) } diff --git a/hack/.packages b/hack/.packages index 336c414f2c..e7b4e08144 100644 --- a/hack/.packages +++ b/hack/.packages @@ -105,6 +105,8 @@ k8s.io/kops/pkg/model k8s.io/kops/pkg/model/alimodel k8s.io/kops/pkg/model/awsmodel k8s.io/kops/pkg/model/components +k8s.io/kops/pkg/model/components/addonmanifests +k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller k8s.io/kops/pkg/model/components/etcdmanager k8s.io/kops/pkg/model/components/kubeapiserver k8s.io/kops/pkg/model/components/node-authorizer diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index f07c48dd81..f0d9d7b007 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -195,6 +195,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie allErrs = append(allErrs, field.Forbidden(fieldPath.Child("iam", "legacy"), "legacy IAM permissions are no longer supported")) } + if (spec.IAM == nil || spec.IAM.Legacy) && featureflag.UseServiceAccountIAM.Enabled() { + allErrs = append(allErrs, field.Forbidden(fieldPath.Child("iam", "legacy"), "legacy IAM permissions are not supported with UseServiceAccountIAM")) + } + if spec.RollingUpdate != nil { allErrs = append(allErrs, validateRollingUpdate(spec.RollingUpdate, fieldPath.Child("rollingUpdate"), false)...) } diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index e2d3c21de8..18e42de960 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -97,6 +97,8 @@ var ( LegacyIAM = New("LegacyIAM", Bool(false)) // ClusterAddons activates experimental cluster-addons support ClusterAddons = New("ClusterAddons", Bool(false)) + // UseServiceAccountIAM controls whether we use pod-level IAM permissions for our system pods. + UseServiceAccountIAM = New("UseServiceAccountIAM", Bool(false)) // PublicJWKS enables public jwks access. This is generally not as secure as republishing. PublicJWKS = New("PublicJWKS", Bool(false)) ) diff --git a/pkg/kubemanifest/manifest.go b/pkg/kubemanifest/manifest.go index 91f19aa06b..3fce7f6c37 100644 --- a/pkg/kubemanifest/manifest.go +++ b/pkg/kubemanifest/manifest.go @@ -19,6 +19,7 @@ package kubemanifest import ( "bytes" "fmt" + "strings" "github.com/ghodss/yaml" "k8s.io/klog/v2" @@ -140,3 +141,58 @@ func (m *Object) APIVersion() string { } return s } + +// Reparse parses a subfield from an object +func (m *Object) Reparse(obj interface{}, fields ...string) error { + humanFields := strings.Join(fields, ".") + + current := m.data + for _, field := range fields { + v, found := current[field] + if !found { + return fmt.Errorf("field %q in %s not found", field, humanFields) + } + + m, ok := v.(map[string]interface{}) + if !ok { + return fmt.Errorf("field %q in %s was not an object, was %T", field, humanFields, v) + } + current = m + } + + b, err := yaml.Marshal(current) + if err != nil { + return fmt.Errorf("error marshaling %s to yaml: %v", humanFields, err) + } + + if err := yaml.Unmarshal(b, obj); err != nil { + return fmt.Errorf("error unmarshaling subobject %s: %v", humanFields, err) + } + + return nil +} + +// Set mutates a subfield to the newValue +func (m *Object) Set(newValue interface{}, fieldPath ...string) error { + humanFields := strings.Join(fieldPath, ".") + + current := m.data + if len(fieldPath) >= 2 { + for _, field := range fieldPath[:len(fieldPath)-1] { + v, found := current[field] + if !found { + return fmt.Errorf("field %q in %s not found", field, humanFields) + } + + m, ok := v.(map[string]interface{}) + if !ok { + return fmt.Errorf("field %q in %s was not an object, was %T", field, humanFields, v) + } + current = m + } + } + + current[fieldPath[len(fieldPath)-1]] = newValue + + return nil +} diff --git a/pkg/model/BUILD.bazel b/pkg/model/BUILD.bazel index 88d37f058e..d2ce3e26c3 100644 --- a/pkg/model/BUILD.bazel +++ b/pkg/model/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "//pkg/pki:go_default_library", "//pkg/rbac:go_default_library", "//pkg/tokens:go_default_library", + "//pkg/util/stringorslice:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/alitasks:go_default_library", "//upup/pkg/fi/cloudup/aliup:go_default_library", diff --git a/pkg/model/alimodel/policy_builder.go b/pkg/model/alimodel/policy_builder.go index 0c50d9c289..2460e1245c 100644 --- a/pkg/model/alimodel/policy_builder.go +++ b/pkg/model/alimodel/policy_builder.go @@ -329,7 +329,12 @@ func (b *PolicyBuilder) AddOSSPermissions(p *Policy) (*Policy, error) { } } - writeablePaths, err := iam.WriteableVFSPaths(b.Cluster, b.Role) + nodeRole, err := iam.BuildNodeRoleSubject(b.Role) + if err != nil { + return nil, err + } + + writeablePaths, err := iam.WriteableVFSPaths(b.Cluster, nodeRole) if err != nil { return nil, err } diff --git a/pkg/model/components/addonmanifests/BUILD.bazel b/pkg/model/components/addonmanifests/BUILD.bazel new file mode 100644 index 0000000000..9d8a42cc24 --- /dev/null +++ b/pkg/model/components/addonmanifests/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["remap.go"], + importpath = "k8s.io/kops/pkg/model/components/addonmanifests", + visibility = ["//visibility:public"], + deps = [ + "//channels/pkg/api:go_default_library", + "//pkg/assets:go_default_library", + "//pkg/kubemanifest:go_default_library", + "//pkg/model:go_default_library", + "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", + "//upup/pkg/fi:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], +) diff --git a/pkg/model/components/addonmanifests/dnscontroller/BUILD.bazel b/pkg/model/components/addonmanifests/dnscontroller/BUILD.bazel new file mode 100644 index 0000000000..a292c2b5a2 --- /dev/null +++ b/pkg/model/components/addonmanifests/dnscontroller/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "iam.go", + "remap.go", + ], + importpath = "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller", + visibility = ["//visibility:public"], + deps = [ + "//channels/pkg/api:go_default_library", + "//pkg/kubemanifest:go_default_library", + "//pkg/model:go_default_library", + "//pkg/model/iam:go_default_library", + "//vendor/github.com/blang/semver/v4:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) diff --git a/pkg/model/components/addonmanifests/dnscontroller/iam.go b/pkg/model/components/addonmanifests/dnscontroller/iam.go new file mode 100644 index 0000000000..7264dce940 --- /dev/null +++ b/pkg/model/components/addonmanifests/dnscontroller/iam.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dnscontroller + +import ( + "k8s.io/apimachinery/pkg/types" + "k8s.io/kops/pkg/model/iam" +) + +// ServiceAccount represents the service-account used by the dns-controller. +// It implements iam.Subject to get AWS IAM permissions. +type ServiceAccount struct { +} + +var _ iam.Subject = &ServiceAccount{} + +// BuildAWSPolicy generates a custom policy for a ServiceAccount IAM role. +func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, error) { + p := &iam.Policy{ + Version: iam.PolicyDefaultVersion, + } + + iam.AddDNSControllerPermissions(b, p) + + return p, nil +} + +// ServiceAccount returns the kubernetes service account used. +func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{ + Namespace: "kube-system", + Name: "dns-controller", + }, true +} diff --git a/pkg/model/components/addonmanifests/dnscontroller/remap.go b/pkg/model/components/addonmanifests/dnscontroller/remap.go new file mode 100644 index 0000000000..aed2e9e1a7 --- /dev/null +++ b/pkg/model/components/addonmanifests/dnscontroller/remap.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dnscontroller + +import ( + "fmt" + + "github.com/blang/semver/v4" + corev1 "k8s.io/api/core/v1" + addonsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/iam" +) + +// Remap remaps the dns-controller addon +func Remap(context *model.KopsModelContext, addon *addonsapi.AddonSpec, objects []*kubemanifest.Object) error { + if !context.UseServiceAccountIAM() { + return nil + } + + if addon.KubernetesVersion != "" { + versionRange, err := semver.ParseRange(addon.KubernetesVersion) + if err != nil { + return fmt.Errorf("cannot parse KubernetesVersion=%q", addon.KubernetesVersion) + } + + if !kubernetesRangesIntersect(versionRange, semver.MustParseRange(">= 1.19.0")) { + // Skip; this is an older manifest + return nil + } + } + + var deployments []*kubemanifest.Object + for _, object := range objects { + if object.Kind() != "Deployment" { + continue + } + if object.APIVersion() != "apps/v1" { + continue + } + deployments = append(deployments, object) + } + + if len(deployments) != 1 { + return fmt.Errorf("expected exactly one Deployment in dns-controller manifest, found %d", len(deployments)) + } + + podSpec := &corev1.PodSpec{} + if err := deployments[0].Reparse(podSpec, "spec", "template", "spec"); err != nil { + return fmt.Errorf("failed to parse spec.template.spec from Deployment: %v", err) + } + + containers := podSpec.Containers + if len(containers) != 1 { + return fmt.Errorf("expected exactly one container in dns-controller Deployment, found %d", len(containers)) + } + container := &containers[0] + + if err := iam.AddServiceAccountRole(&context.IAMModelContext, podSpec, container, &ServiceAccount{}); err != nil { + return err + } + + if err := deployments[0].Set(podSpec, "spec", "template", "spec"); err != nil { + return err + } + + return nil +} + +// kubernetesRangesIntersect returns true if the two semver ranges overlap +// Sadly there's no actual function to do this. +// Instead we restrict to kubernetes versions, and just probe with 1.1, 1.2, 1.3 etc. +// This will therefore be inaccurate if there's a patch specifier +func kubernetesRangesIntersect(r1, r2 semver.Range) bool { + for minor := 1; minor < 99; minor++ { + v := semver.Version{Major: 1, Minor: uint64(minor), Patch: 0} + if r1(v) && r2(v) { + return true + } + } + return false +} diff --git a/pkg/model/components/addonmanifests/remap.go b/pkg/model/components/addonmanifests/remap.go new file mode 100644 index 0000000000..2880b2af91 --- /dev/null +++ b/pkg/model/components/addonmanifests/remap.go @@ -0,0 +1,67 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addonmanifests + +import ( + "fmt" + + "k8s.io/klog/v2" + addonsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" + "k8s.io/kops/upup/pkg/fi" +) + +func RemapAddonManifest(addon *addonsapi.AddonSpec, context *model.KopsModelContext, assetBuilder *assets.AssetBuilder, manifest []byte) ([]byte, error) { + name := fi.StringValue(addon.Name) + + { + objects, err := kubemanifest.LoadObjectsFrom(manifest) + if err != nil { + return nil, err + } + + remapped := false + if name == "dns-controller.addons.k8s.io" { + if err := dnscontroller.Remap(context, addon, objects); err != nil { + return nil, err + } + remapped = true + } + + if remapped { + b, err := objects.ToYAML() + if err != nil { + return nil, err + } + manifest = b + } + } + + { + remapped, err := assetBuilder.RemapManifest(manifest) + if err != nil { + klog.Infof("invalid manifest: %s", string(manifest)) + return nil, fmt.Errorf("error remapping manifest %s: %v", manifest, err) + } + manifest = remapped + } + + return manifest, nil +} diff --git a/pkg/model/context.go b/pkg/model/context.go index b4f5813349..6eedce2a17 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -408,3 +408,8 @@ func (m *KopsModelContext) NodePortRange() (utilnet.PortRange, error) { return defaultServiceNodePortRange, nil } + +// UseServiceAccountIAM returns true if we are using service-account bound IAM roles. +func (m *KopsModelContext) UseServiceAccountIAM() bool { + return featureflag.UseServiceAccountIAM.Enabled() && m.IsKubernetesGTE("1.12") +} diff --git a/pkg/model/gcemodel/autoscalinggroup.go b/pkg/model/gcemodel/autoscalinggroup.go index 23a9c6d964..38801caccd 100644 --- a/pkg/model/gcemodel/autoscalinggroup.go +++ b/pkg/model/gcemodel/autoscalinggroup.go @@ -97,7 +97,12 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { }, } - storagePaths, err := iam.WriteableVFSPaths(b.Cluster, ig.Spec.Role) + nodeRole, err := iam.BuildNodeRoleSubject(ig.Spec.Role) + if err != nil { + return err + } + + storagePaths, err := iam.WriteableVFSPaths(b.Cluster, nodeRole) if err != nil { return err } diff --git a/pkg/model/gcemodel/storageacl.go b/pkg/model/gcemodel/storageacl.go index f545a70913..aef69df90d 100644 --- a/pkg/model/gcemodel/storageacl.go +++ b/pkg/model/gcemodel/storageacl.go @@ -68,8 +68,12 @@ func (b *StorageAclBuilder) Build(c *fi.ModelBuilderContext) error { } klog.Warningf("we need to split master / node roles") - role := kops.InstanceGroupRoleMaster - writeablePaths, err := iam.WriteableVFSPaths(b.Cluster, role) + nodeRole, err := iam.BuildNodeRoleSubject(kops.InstanceGroupRoleMaster) + if err != nil { + return err + } + + writeablePaths, err := iam.WriteableVFSPaths(b.Cluster, nodeRole) if err != nil { return err } diff --git a/pkg/model/iam.go b/pkg/model/iam.go index bef02fae1c..b228fa3098 100644 --- a/pkg/model/iam.go +++ b/pkg/model/iam.go @@ -19,11 +19,11 @@ package model import ( "fmt" "strings" - "text/template" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model/iam" + "k8s.io/kops/pkg/util/stringorslice" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" ) @@ -37,7 +37,7 @@ type IAMModelBuilder struct { var _ fi.ModelBuilder = &IAMModelBuilder{} -const RolePolicyTemplate = `{ +const NodeRolePolicyTemplate = `{ "Version": "2012-10-17", "Statement": [ { @@ -72,11 +72,16 @@ func (b *IAMModelBuilder) Build(c *fi.ModelBuilderContext) error { // Generate IAM tasks for each shared role for profileARN, igRole := range sharedProfileARNsToIGRole { + role, err := iam.BuildNodeRoleSubject(igRole) + if err != nil { + return err + } + iamName, err := findCustomAuthNameFromArn(profileARN) if err != nil { return fmt.Errorf("unable to parse instance profile name from arn %q: %v", profileARN, err) } - err = b.buildIAMTasks(igRole, iamName, c, true) + err = b.buildIAMTasks(role, iamName, c, true) if err != nil { return err } @@ -84,70 +89,136 @@ func (b *IAMModelBuilder) Build(c *fi.ModelBuilderContext) error { // Generate IAM tasks for each managed role for igRole := range managedRoles { - iamName := b.IAMName(igRole) - err := b.buildIAMTasks(igRole, iamName, c, false) + role, err := iam.BuildNodeRoleSubject(igRole) if err != nil { return err } + + iamName := b.IAMName(igRole) + if err := b.buildIAMTasks(role, iamName, c, false); err != nil { + return err + } } return nil } -func (b *IAMModelBuilder) buildIAMTasks(igRole kops.InstanceGroupRole, iamName string, c *fi.ModelBuilderContext, shared bool) error { - { // To minimize diff for easier code review - var iamRole *awstasks.IAMRole - { - rolePolicy, err := b.buildAWSIAMRolePolicy() - if err != nil { - return err - } +// BuildServiceAccountRoleTasks build tasks specifically for the ServiceAccount role. +func (b *IAMModelBuilder) BuildServiceAccountRoleTasks(role iam.Subject, c *fi.ModelBuilderContext) error { + iamName, err := b.IAMNameForServiceAccountRole(role) + if err != nil { + return err + } - iamRole = &awstasks.IAMRole{ - Name: s(iamName), - Lifecycle: b.Lifecycle, + iamRole, err := b.buildIAMRole(role, iamName, c) + if err != nil { + return err + } - RolePolicyDocument: fi.WrapResource(rolePolicy), - ExportWithID: s(strings.ToLower(string(igRole)) + "s"), - } + if err := b.buildIAMRolePolicy(role, iamName, iamRole, c); err != nil { + return err + } - if b.Cluster.Spec.IAM != nil && b.Cluster.Spec.IAM.PermissionsBoundary != nil { - iamRole.PermissionsBoundary = b.Cluster.Spec.IAM.PermissionsBoundary - } + return nil +} - c.AddTask(iamRole) +func (b *IAMModelBuilder) buildIAMRole(role iam.Subject, iamName string, c *fi.ModelBuilderContext) (*awstasks.IAMRole, error) { + roleKey, isServiceAccount := b.roleKey(role) - } + rolePolicy, err := b.buildAWSIAMRolePolicy(role) + if err != nil { + return nil, err + } - { - iamPolicy := &iam.PolicyResource{ - Builder: &iam.PolicyBuilder{ - Cluster: b.Cluster, - Role: igRole, - Region: b.Region, - }, - } + iamRole := &awstasks.IAMRole{ + Name: s(iamName), + Lifecycle: b.Lifecycle, - // This is slightly tricky; we need to know the hosted zone id, - // but we might be creating the hosted zone dynamically. + RolePolicyDocument: fi.WrapResource(rolePolicy), + } - // TODO: I don't love this technique for finding the task by name & modifying it - dnsZoneTask, found := c.Tasks["DNSZone/"+b.NameForDNSZone()] - if found { - iamPolicy.DNSZone = dnsZoneTask.(*awstasks.DNSZone) - } else { - klog.V(2).Infof("Task %q not found; won't set route53 permissions in IAM", "DNSZone/"+b.NameForDNSZone()) - } + if isServiceAccount { + // e.g. kube-system-dns-controller + iamRole.ExportWithID = s(roleKey) + } else { + // e.g. nodes + iamRole.ExportWithID = s(roleKey + "s") + } - t := &awstasks.IAMRolePolicy{ - Name: s(iamName), - Lifecycle: b.Lifecycle, + if b.Cluster.Spec.IAM != nil && b.Cluster.Spec.IAM.PermissionsBoundary != nil { + iamRole.PermissionsBoundary = b.Cluster.Spec.IAM.PermissionsBoundary + } - Role: iamRole, - PolicyDocument: iamPolicy, - } - c.AddTask(t) - } + c.AddTask(iamRole) + + return iamRole, nil +} + +func (b *IAMModelBuilder) buildIAMRolePolicy(role iam.Subject, iamName string, iamRole *awstasks.IAMRole, c *fi.ModelBuilderContext) error { + iamPolicy := &iam.PolicyResource{ + Builder: &iam.PolicyBuilder{ + Cluster: b.Cluster, + Role: role, + Region: b.Region, + UseServiceAccountIAM: b.UseServiceAccountIAM(), + }, + } + + // This is slightly tricky; we need to know the hosted zone id, + // but we might be creating the hosted zone dynamically. + // We create a stub-reference which will be combined by the execution engine. + iamPolicy.DNSZone = &awstasks.DNSZone{ + Name: fi.String(b.NameForDNSZone()), + } + + t := &awstasks.IAMRolePolicy{ + Name: s(iamName), + Lifecycle: b.Lifecycle, + + Role: iamRole, + PolicyDocument: iamPolicy, + } + c.AddTask(t) + + return nil +} + +// roleKey builds a string to represent the role uniquely. It returns true if this is a service account role. +func (b *IAMModelBuilder) roleKey(role iam.Subject) (string, bool) { + serviceAccount, ok := role.ServiceAccount() + if ok { + return strings.ToLower(serviceAccount.Namespace + "-" + serviceAccount.Name), true + } + + // This isn't great, but we have to be backwards compatible with the old names. + switch role.(type) { + case *iam.NodeRoleMaster: + return strings.ToLower(string(kops.InstanceGroupRoleMaster)), false + case *iam.NodeRoleNode: + return strings.ToLower(string(kops.InstanceGroupRoleNode)), false + case *iam.NodeRoleBastion: + return strings.ToLower(string(kops.InstanceGroupRoleBastion)), false + + default: + klog.Fatalf("unknown node role type: %T", role) + return "", false + } +} + +func (b *IAMModelBuilder) buildIAMTasks(role iam.Subject, iamName string, c *fi.ModelBuilderContext, shared bool) error { + roleKey, _ := b.roleKey(role) + + iamRole, err := b.buildIAMRole(role, iamName, c) + if err != nil { + return err + } + + if err := b.buildIAMRolePolicy(role, iamName, iamRole, c); err != nil { + return err + } + + { + // To minimize diff for easier code review var iamInstanceProfile *awstasks.IAMInstanceProfile { @@ -176,10 +247,10 @@ func (b *IAMModelBuilder) buildIAMTasks(igRole kops.InstanceGroupRole, iamName s if b.Cluster.Spec.ExternalPolicies != nil { p := *(b.Cluster.Spec.ExternalPolicies) - externalPolicies = append(externalPolicies, p[strings.ToLower(string(igRole))]...) + externalPolicies = append(externalPolicies, p[roleKey]...) } - name := fmt.Sprintf("%s-policyoverride", strings.ToLower(string(igRole))) + name := fmt.Sprintf("%s-policyoverride", roleKey) t := &awstasks.IAMRolePolicy{ Name: s(name), Lifecycle: b.Lifecycle, @@ -197,7 +268,7 @@ func (b *IAMModelBuilder) buildIAMTasks(igRole kops.InstanceGroupRole, iamName s if b.Cluster.Spec.AdditionalPolicies != nil { additionalPolicies := *(b.Cluster.Spec.AdditionalPolicies) - additionalPolicy = additionalPolicies[strings.ToLower(string(igRole))] + additionalPolicy = additionalPolicies[roleKey] } additionalPolicyName := "additional." + iamName @@ -216,7 +287,7 @@ func (b *IAMModelBuilder) buildIAMTasks(igRole kops.InstanceGroupRole, iamName s statements, err := iam.ParseStatements(additionalPolicy) if err != nil { - return fmt.Errorf("additionalPolicy %q is invalid: %v", strings.ToLower(string(igRole)), err) + return fmt.Errorf("additionalPolicy %q is invalid: %v", roleKey, err) } p.Statement = append(p.Statement, statements...) @@ -234,29 +305,61 @@ func (b *IAMModelBuilder) buildIAMTasks(igRole kops.InstanceGroupRole, iamName s c.AddTask(t) } } + return nil } -// buildAWSIAMRolePolicy produces the AWS IAM role policy for the given role -func (b *IAMModelBuilder) buildAWSIAMRolePolicy() (fi.Resource, error) { - functions := template.FuncMap{ - "IAMServiceEC2": func() string { - // IAMServiceEC2 returns the name of the IAM service for EC2 in the current region - // it is ec2.amazonaws.com everywhere but in cn-north, where it is ec2.amazonaws.com.cn - switch b.Region { - case "cn-north-1": - return "ec2.amazonaws.com.cn" - case "cn-northwest-1": - return "ec2.amazonaws.com.cn" - default: - return "ec2.amazonaws.com" - } - }, +// IAMServiceEC2 returns the name of the IAM service for EC2 in the current region. +// It is ec2.amazonaws.com everywhere but in cn-north / cn-northwest, where it is ec2.amazonaws.com.cn +func IAMServiceEC2(region string) string { + switch region { + case "cn-north-1": + return "ec2.amazonaws.com.cn" + case "cn-northwest-1": + return "ec2.amazonaws.com.cn" + default: + return "ec2.amazonaws.com" + } +} + +// buildAWSIAMRolePolicy produces the AWS IAM role policy for the given role. +func (b *IAMModelBuilder) buildAWSIAMRolePolicy(role iam.Subject) (fi.Resource, error) { + var policy string + serviceAccount, ok := role.ServiceAccount() + if ok { + serviceAccountIssuer, err := iam.ServiceAccountIssuer(b.ClusterName(), &b.Cluster.Spec) + if err != nil { + return nil, err + } + oidcProvider := strings.TrimPrefix(serviceAccountIssuer, "https://") + + iamPolicy := &iam.Policy{ + Version: iam.PolicyDefaultVersion, + Statement: []*iam.Statement{ + { + Effect: "Allow", + Principal: iam.Principal{ + Federated: "arn:aws:iam::" + b.AWSAccountID + ":oidc-provider/" + oidcProvider, + }, + Action: stringorslice.String("sts:AssumeRoleWithWebIdentity"), + Condition: map[string]interface{}{ + "StringEquals": map[string]interface{}{ + oidcProvider + ":sub": "system:serviceaccount:" + serviceAccount.Namespace + ":" + serviceAccount.Name, + }, + }, + }, + }, + } + s, err := iamPolicy.AsJSON() + if err != nil { + return nil, err + } + policy = s + } else { + // We don't generate using json.Marshal here, it would create whitespace changes in the policy for existing clusters. + + policy = strings.ReplaceAll(NodeRolePolicyTemplate, "{{ IAMServiceEC2 }}", IAMServiceEC2(b.Region)) } - templateResource, err := NewTemplateResource("AWSIAMRolePolicy", RolePolicyTemplate, functions, nil) - if err != nil { - return nil, err - } - return templateResource, nil + return fi.NewStringResource(policy), nil } diff --git a/pkg/model/iam/BUILD.bazel b/pkg/model/iam/BUILD.bazel index 6a60ed2d75..03736204aa 100644 --- a/pkg/model/iam/BUILD.bazel +++ b/pkg/model/iam/BUILD.bazel @@ -4,7 +4,7 @@ go_library( name = "go_default_library", srcs = [ "iam_builder.go", - "pod_roles.go", + "subject.go", "types.go", ], importpath = "k8s.io/kops/pkg/model/iam", @@ -14,9 +14,12 @@ go_library( "//pkg/apis/kops/model:go_default_library", "//pkg/featureflag:go_default_library", "//pkg/util/stringorslice:go_default_library", + "//pkg/wellknownusers:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", "//util/pkg/vfs:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ], diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index e7c921f0f0..50905b297f 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -35,9 +35,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" - "k8s.io/kops/pkg/apis/kops/model" - "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/util/stringorslice" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -78,9 +77,110 @@ type Condition map[string]interface{} // http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Statement type Statement struct { Effect StatementEffect + Principal Principal Action stringorslice.StringOrSlice Resource stringorslice.StringOrSlice - Condition Condition `json:",omitempty"` + Condition Condition +} + +type jsonWriter struct { + w io.Writer + err error +} + +func (j *jsonWriter) Error() error { + return j.err +} + +func (j *jsonWriter) WriteLiteral(b []byte) { + if j.err != nil { + return + } + _, err := j.w.Write(b) + if err != nil { + j.err = err + } +} + +func (j *jsonWriter) StartObject() { + j.WriteLiteral([]byte("{")) +} + +func (j *jsonWriter) EndObject() { + j.WriteLiteral([]byte("}")) +} + +func (j *jsonWriter) Comma() { + j.WriteLiteral([]byte(",")) +} + +func (j *jsonWriter) Field(s string) { + if j.err != nil { + return + } + b, err := json.Marshal(s) + if err != nil { + j.err = err + return + } + j.WriteLiteral(b) + j.WriteLiteral([]byte(": ")) +} + +func (j *jsonWriter) Marshal(v interface{}) { + if j.err != nil { + return + } + b, err := json.Marshal(v) + if err != nil { + j.err = err + return + } + j.WriteLiteral(b) +} + +// MarshalJSON formats the IAM statement for the AWS IAM restrictions. +// For example, `Resource: []` is not allowed, but golang would force us to use pointers. +func (s *Statement) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + + jw := &jsonWriter{w: &b} + jw.StartObject() + jw.Field("Effect") + jw.Marshal(s.Effect) + + if !s.Principal.IsEmpty() { + jw.Comma() + jw.Field("Principal") + jw.Marshal(s.Principal) + } + if !s.Action.IsEmpty() { + jw.Comma() + jw.Field("Action") + jw.Marshal(s.Action) + } + if !s.Resource.IsEmpty() { + jw.Comma() + jw.Field("Resource") + jw.Marshal(s.Resource) + } + if len(s.Condition) != 0 { + jw.Comma() + jw.Field("Condition") + jw.Marshal(s.Condition) + } + jw.EndObject() + + return b.Bytes(), jw.Error() +} + +type Principal struct { + Federated string `json:",omitempty"` + Service string `json:",omitempty"` +} + +func (p *Principal) IsEmpty() bool { + return *p == Principal{} } // Equal compares two IAM Statements and returns a bool @@ -101,20 +201,18 @@ func (l *Statement) Equal(r *Statement) bool { // PolicyBuilder struct defines all valid fields to be used when building the // AWS IAM policy document for a given instance group role. type PolicyBuilder struct { - Cluster *kops.Cluster - HostedZoneID string - KMSKeys []string - Region string - ResourceARN *string - Role kops.InstanceGroupRole + Cluster *kops.Cluster + HostedZoneID string + KMSKeys []string + Region string + ResourceARN *string + Role Subject + UseServiceAccountIAM bool } // BuildAWSPolicy builds a set of IAM policy statements based on the // instance group type and IAM Legacy flag within the Cluster Spec func (b *PolicyBuilder) BuildAWSPolicy() (*Policy, error) { - var p *Policy - var err error - // Retrieve all the KMS Keys in use for _, e := range b.Cluster.Spec.EtcdClusters { for _, m := range e.Members { @@ -124,31 +222,16 @@ func (b *PolicyBuilder) BuildAWSPolicy() (*Policy, error) { } } - switch b.Role { - case kops.InstanceGroupRoleBastion: - p, err = b.BuildAWSPolicyBastion() - if err != nil { - return nil, fmt.Errorf("failed to generate AWS IAM Policy for Bastion Instance Group: %v", err) - } - case kops.InstanceGroupRoleNode: - p, err = b.BuildAWSPolicyNode() - if err != nil { - return nil, fmt.Errorf("failed to generate AWS IAM Policy for Node Instance Group: %v", err) - } - case kops.InstanceGroupRoleMaster: - p, err = b.BuildAWSPolicyMaster() - if err != nil { - return nil, fmt.Errorf("failed to generate AWS IAM Policy for Master Instance Group: %v", err) - } - default: - return nil, fmt.Errorf("unrecognised instance group type: %s", b.Role) + p, err := b.Role.BuildAWSPolicy(b) + if err != nil { + return nil, fmt.Errorf("failed to generate AWS IAM Policy: %v", err) } return p, nil } -// BuildAWSPolicyMaster generates a custom policy for a Kubernetes master. -func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) { +// BuildAWSPolicy generates a custom policy for a Kubernetes master. +func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { resource := createResource(b) p := &Policy{ @@ -169,7 +252,12 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) { addKMSIAMPolicies(p, stringorslice.Slice(b.KMSKeys), b.Cluster.Spec.IAM.Legacy) } - b.addRoute53Permissions(p) + if !b.UseServiceAccountIAM { + if b.Cluster.Spec.IAM.Legacy { + addLegacyDNSControllerPermissions(b, p) + } + AddDNSControllerPermissions(b, p) + } if b.Cluster.Spec.IAM.Legacy || b.Cluster.Spec.IAM.AllowContainerRegistry { addECRPermissions(p) @@ -190,8 +278,8 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) { return p, nil } -// BuildAWSPolicyNode generates a custom policy for a Kubernetes node. -func (b *PolicyBuilder) BuildAWSPolicyNode() (*Policy, error) { +// BuildAWSPolicy generates a custom policy for a Kubernetes node. +func (r *NodeRoleNode) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { resource := createResource(b) p := &Policy{ @@ -205,7 +293,10 @@ func (b *PolicyBuilder) BuildAWSPolicyNode() (*Policy, error) { return nil, fmt.Errorf("failed to generate AWS IAM S3 access statements: %v", err) } - b.addRoute53Permissions(p) + if !b.UseServiceAccountIAM && b.Cluster.Spec.IAM.Legacy { + addLegacyDNSControllerPermissions(b, p) + AddDNSControllerPermissions(b, p) + } if b.Cluster.Spec.IAM.Legacy || b.Cluster.Spec.IAM.AllowContainerRegistry { addECRPermissions(p) @@ -222,8 +313,8 @@ func (b *PolicyBuilder) BuildAWSPolicyNode() (*Policy, error) { return p, nil } -// BuildAWSPolicyBastion generates a custom policy for a bastion host. -func (b *PolicyBuilder) BuildAWSPolicyBastion() (*Policy, error) { +// BuildAWSPolicy generates a custom policy for a bastion host. +func (r *NodeRoleBastion) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { resource := createResource(b) p := &Policy{ @@ -258,8 +349,8 @@ func (b *PolicyBuilder) IAMPrefix() string { } } -// AddS3Permissions updates an IAM Policy with statements granting tailored -// access to S3 assets, depending on the instance group role +// AddS3Permissions builds an IAM Policy, with statements granting tailored +// access to S3 assets, depending on the instance group or service-account role func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) { // For S3 IAM permissions we grant permissions to subtrees, so find the parents; // we don't need to grant mypath and mypath/child. @@ -330,18 +421,20 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) { return nil, err } - sort.Strings(resources) + if len(resources) != 0 { + sort.Strings(resources) - // Add the prefix for IAM - for i, r := range resources { - resources[i] = b.IAMPrefix() + ":s3:::" + iamS3Path + r + // Add the prefix for IAM + for i, r := range resources { + resources[i] = b.IAMPrefix() + ":s3:::" + iamS3Path + r + } + + p.Statement = append(p.Statement, &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{"s3:Get*"}), + Resource: stringorslice.Of(resources...), + }) } - - p.Statement = append(p.Statement, &Statement{ - Effect: StatementEffectAllow, - Action: stringorslice.Slice([]string{"s3:Get*"}), - Resource: stringorslice.Of(resources...), - }) } } else if _, ok := vfsPath.(*vfs.MemFSPath); ok { // Tests -ignore - nothing we can do in terms of IAM policy @@ -404,11 +497,12 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) { return p, nil } -func WriteableVFSPaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]vfs.Path, error) { +func WriteableVFSPaths(cluster *kops.Cluster, role Subject) ([]vfs.Path, error) { var paths []vfs.Path - // On the master, grant IAM permissions to the backup store, if it is configured - if role == kops.InstanceGroupRoleMaster { + // etcd-manager needs write permissions to the backup store + switch role.(type) { + case *NodeRoleMaster: backupStores := sets.NewString() for _, c := range cluster.Spec.EtcdClusters { if c.Backups == nil || c.Backups.BackupStore == "" || backupStores.Has(c.Backups.BackupStore) { @@ -426,16 +520,19 @@ func WriteableVFSPaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]vf backupStores.Insert(backupStore) } } + return paths, nil } // ReadableStatePaths returns the file paths that should be readable in the cluster's state store "directory" -func ReadableStatePaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]string, error) { +func ReadableStatePaths(cluster *kops.Cluster, role Subject) ([]string, error) { var paths []string - if role == kops.InstanceGroupRoleMaster { + switch role.(type) { + case *NodeRoleMaster: paths = append(paths, "/*") - } else if role == kops.InstanceGroupRoleNode { + + case *NodeRoleNode: paths = append(paths, "/addons/*", "/cluster.spec", @@ -570,22 +667,20 @@ func addECRPermissions(p *Policy) { }) } -func (b *PolicyBuilder) addRoute53Permissions(p *Policy) { - // Only the master (unless in legacy mode) - if b.Role != kops.InstanceGroupRoleMaster && !b.Cluster.Spec.IAM.Legacy { - return - } - +// addLegacyDNSControllerPermissions adds legacy IAM permissions used by the node roles. +func addLegacyDNSControllerPermissions(b *PolicyBuilder, p *Policy) { // Legacy IAM permissions for node roles - if b.Cluster.Spec.IAM.Legacy { - wildcard := stringorslice.Slice([]string{"*"}) - p.Statement = append(p.Statement, &Statement{ - Effect: StatementEffectAllow, - Action: stringorslice.Slice([]string{"route53:ListHostedZones"}), - Resource: wildcard, - }) - } + wildcard := stringorslice.Slice([]string{"*"}) + p.Statement = append(p.Statement, &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{"route53:ListHostedZones"}), + Resource: wildcard, + }) +} +// AddDNSControllerPermissions adds IAM permissions used by the dns-controller. +// TODO: Move this to dnscontroller, but it requires moving a lot of code around. +func AddDNSControllerPermissions(b *PolicyBuilder, p *Policy) { // Permissions to mutate the specific zone if b.HostedZoneID == "" { return diff --git a/pkg/model/iam/iam_builder_test.go b/pkg/model/iam/iam_builder_test.go index 6ab891f11f..e07bd3e7e6 100644 --- a/pkg/model/iam/iam_builder_test.go +++ b/pkg/model/iam/iam_builder_test.go @@ -48,11 +48,31 @@ func TestRoundTrip(t *testing.T) { }, JSON: "{\"Effect\":\"Deny\",\"Action\":[\"ec2:DescribeRegions\",\"ec2:DescribeInstances\"],\"Resource\":[\"a\",\"b\"]}", }, + { + IAM: &Statement{ + Effect: StatementEffectDeny, + Principal: Principal{Federated: "federated"}, + Condition: map[string]interface{}{ + "foo": 1, + }, + }, + JSON: "{\"Effect\":\"Deny\",\"Principal\":{\"Federated\":\"federated\"},\"Condition\":{\"foo\":1}}", + }, + { + IAM: &Statement{ + Effect: StatementEffectDeny, + Principal: Principal{Service: "service"}, + Condition: map[string]interface{}{ + "bar": "baz", + }, + }, + JSON: "{\"Effect\":\"Deny\",\"Principal\":{\"Service\":\"service\"},\"Condition\":{\"bar\":\"baz\"}}", + }, } for _, g := range grid { actualJSON, err := json.Marshal(g.IAM) if err != nil { - t.Errorf("error encoding IAM %s to json: %v", g.IAM, err) + t.Errorf("error encoding IAM %v to json: %v", g.IAM, err) } if g.JSON != string(actualJSON) { @@ -74,61 +94,61 @@ func TestRoundTrip(t *testing.T) { func TestPolicyGeneration(t *testing.T) { grid := []struct { - Role kops.InstanceGroupRole + Role Subject LegacyIAM bool AllowContainerRegistry bool Policy string }{ { - Role: "Master", + Role: &NodeRoleMaster{}, LegacyIAM: true, AllowContainerRegistry: false, Policy: "tests/iam_builder_master_legacy.json", }, { - Role: "Master", + Role: &NodeRoleMaster{}, LegacyIAM: false, AllowContainerRegistry: false, Policy: "tests/iam_builder_master_strict.json", }, { - Role: "Master", + Role: &NodeRoleMaster{}, LegacyIAM: false, AllowContainerRegistry: true, Policy: "tests/iam_builder_master_strict_ecr.json", }, { - Role: "Node", + Role: &NodeRoleNode{}, LegacyIAM: true, AllowContainerRegistry: false, Policy: "tests/iam_builder_node_legacy.json", }, { - Role: "Node", + Role: &NodeRoleNode{}, LegacyIAM: false, AllowContainerRegistry: false, Policy: "tests/iam_builder_node_strict.json", }, { - Role: "Node", + Role: &NodeRoleNode{}, LegacyIAM: false, AllowContainerRegistry: true, Policy: "tests/iam_builder_node_strict_ecr.json", }, { - Role: "Bastion", + Role: &NodeRoleBastion{}, LegacyIAM: true, AllowContainerRegistry: false, Policy: "tests/iam_builder_bastion.json", }, { - Role: "Bastion", + Role: &NodeRoleBastion{}, LegacyIAM: false, AllowContainerRegistry: false, Policy: "tests/iam_builder_bastion.json", }, { - Role: "Bastion", + Role: &NodeRoleBastion{}, LegacyIAM: false, AllowContainerRegistry: true, Policy: "tests/iam_builder_bastion.json", diff --git a/pkg/model/iam/pod_roles.go b/pkg/model/iam/pod_roles.go deleted file mode 100644 index d8bf104829..0000000000 --- a/pkg/model/iam/pod_roles.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iam - -import ( - "fmt" - - "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/featureflag" -) - -// ServiceAccountIssuer determines the issuer in the ServiceAccount JWTs -func ServiceAccountIssuer(clusterName string, clusterSpec *kops.ClusterSpec) (string, error) { - if featureflag.PublicJWKS.Enabled() { - return "https://api." + clusterName, nil - } - - return "", fmt.Errorf("ServiceAcccountIssuer not (currently) supported without PublicJWKS") -} diff --git a/pkg/model/iam/subject.go b/pkg/model/iam/subject.go new file mode 100644 index 0000000000..ab61c6704b --- /dev/null +++ b/pkg/model/iam/subject.go @@ -0,0 +1,162 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iam + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/wellknownusers" +) + +// Subject represents an IAM identity, to which permissions are granted. +// It is implemented by NodeRole objects and per-ServiceAccount objects. +type Subject interface { + // BuildAWSPolicy builds the AWS permissions for the given subject. + BuildAWSPolicy(*PolicyBuilder) (*Policy, error) + + // ServiceAccount returns the kubernetes service account used by pods with this specified role. + // For node roles, it returns an empty NamespacedName and false. + ServiceAccount() (types.NamespacedName, bool) +} + +// NodeRoleMaster represents the role of control-plane nodes, and implements Subject. +type NodeRoleMaster struct { +} + +// ServiceAccount implements Subject. +func (_ *NodeRoleMaster) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{}, false +} + +// NodeRoleNode represents the role of normal ("worker") nodes, and implements Subject. +type NodeRoleNode struct { +} + +// ServiceAccount implements Subject. +func (_ *NodeRoleNode) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{}, false +} + +// NodeRoleNode represents the role of bastion nodes, and implements Subject. +type NodeRoleBastion struct { +} + +// ServiceAccount implements Subject. +func (_ *NodeRoleBastion) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{}, false +} + +// BuildNodeRoleSubject returns a Subject implementation for the specified InstanceGroupRole. +func BuildNodeRoleSubject(igRole kops.InstanceGroupRole) (Subject, error) { + switch igRole { + case kops.InstanceGroupRoleMaster: + return &NodeRoleMaster{}, nil + + case kops.InstanceGroupRoleNode: + return &NodeRoleNode{}, nil + + case kops.InstanceGroupRoleBastion: + return &NodeRoleBastion{}, nil + + default: + return nil, fmt.Errorf("unknown instancegroup role %q", igRole) + } +} + +// ServiceAccountIssuer determines the issuer in the ServiceAccount JWTs +func ServiceAccountIssuer(clusterName string, clusterSpec *kops.ClusterSpec) (string, error) { + if featureflag.PublicJWKS.Enabled() { + return "https://api." + clusterName, nil + } + + return "", fmt.Errorf("ServiceAcccountIssuer not (currently) supported without PublicJWKS") +} + +// AddServiceAccountRole adds the appropriate mounts / env vars to enable a pod to use a service-account role +func AddServiceAccountRole(context *IAMModelContext, podSpec *corev1.PodSpec, container *corev1.Container, serviceAccountRole Subject) error { + cloudProvider := kops.CloudProviderID(context.Cluster.Spec.CloudProvider) + + switch cloudProvider { + case kops.CloudProviderAWS: + return addServiceAccountRoleForAWS(context, podSpec, container, serviceAccountRole) + default: + return fmt.Errorf("ServiceAccount-level IAM is not yet supported on cloud %T", cloudProvider) + } +} + +func addServiceAccountRoleForAWS(context *IAMModelContext, podSpec *corev1.PodSpec, container *corev1.Container, serviceAccountRole Subject) error { + roleName, err := context.IAMNameForServiceAccountRole(serviceAccountRole) + if err != nil { + return err + } + + awsRoleARN := "arn:aws:iam::" + context.AWSAccountID + ":role/" + roleName + tokenDir := "/var/run/secrets/amazonaws.com/" + tokenName := "token" + + volume := corev1.Volume{ + Name: "token-amazonaws-com", + } + + mode := int32(0o644) + expiration := int64(86400) + volume.Projected = &corev1.ProjectedVolumeSource{ + DefaultMode: &mode, + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "amazonaws.com", + ExpirationSeconds: &expiration, + Path: tokenName, + }, + }, + }, + } + podSpec.Volumes = append(podSpec.Volumes, volume) + + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + MountPath: tokenDir, + Name: volume.Name, + ReadOnly: true, + }) + + container.Env = append(container.Env, corev1.EnvVar{ + Name: "AWS_ROLE_ARN", + Value: awsRoleARN, + }) + + container.Env = append(container.Env, corev1.EnvVar{ + Name: "AWS_WEB_IDENTITY_TOKEN_FILE", + Value: tokenDir + tokenName, + }) + + // Set securityContext.fsGroup to enable file to be read + // background: https://github.com/kubernetes/enhancements/pull/1598 + if podSpec.SecurityContext == nil { + podSpec.SecurityContext = &corev1.PodSecurityContext{} + } + if podSpec.SecurityContext.FSGroup == nil { + fsGroup := int64(wellknownusers.Generic) + podSpec.SecurityContext.FSGroup = &fsGroup + } + + return nil +} diff --git a/pkg/model/iam/types.go b/pkg/model/iam/types.go index 7dcfeb1c0b..5feb634a00 100644 --- a/pkg/model/iam/types.go +++ b/pkg/model/iam/types.go @@ -33,9 +33,22 @@ func ParseStatements(policy string) ([]*Statement, error) { } type IAMModelContext struct { + // AWSAccountID holds the 12 digit AWS account ID, when running on AWS + AWSAccountID string + Cluster *kops.Cluster } +// IAMNameForServiceAccountRole determines the name of the IAM Role and Instance Profile to use for the service-account role +func (b *IAMModelContext) IAMNameForServiceAccountRole(role Subject) (string, error) { + serviceAccount, ok := role.ServiceAccount() + if !ok { + return "", fmt.Errorf("role %v does not have ServiceAccount", role) + } + + return serviceAccount.Name + "." + serviceAccount.Namespace + ".sa." + b.ClusterName(), nil +} + // ClusterName returns the cluster name func (b *IAMModelContext) ClusterName() string { return b.Cluster.ObjectMeta.Name diff --git a/pkg/model/manifests.go b/pkg/model/manifests.go index 8a31a3be2e..2e1c932424 100644 --- a/pkg/model/manifests.go +++ b/pkg/model/manifests.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog/v2" ) -// ParseManifest parses a set of objects from a []byte +// ParseManifest parses a typed set of objects from a []byte func ParseManifest(data []byte) ([]runtime.Object, error) { decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) deser := scheme.Codecs.UniversalDeserializer() diff --git a/pkg/util/stringorslice/stringorslice.go b/pkg/util/stringorslice/stringorslice.go index 0d04fc9588..7b88f40e24 100644 --- a/pkg/util/stringorslice/stringorslice.go +++ b/pkg/util/stringorslice/stringorslice.go @@ -27,6 +27,10 @@ type StringOrSlice struct { forceEncodeAsArray bool } +func (s *StringOrSlice) IsEmpty() bool { + return len(s.values) == 0 +} + // Slice will build a value that marshals to a JSON array func Slice(v []string) StringOrSlice { return StringOrSlice{values: v, forceEncodeAsArray: true} diff --git a/tests/integration/update_cluster/public-jwks/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..256ce24ad2 --- /dev/null +++ b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/api.minimal.example.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "api.minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller" + } + } + } + ] +} diff --git a/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..212e07037d --- /dev/null +++ b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,34 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:GetChange" + ], + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ListHostedZones" + ], + "Resource": [ + "*" + ] + } + ] +} diff --git a/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy index 107ccaf8e6..0081268038 100644 --- a/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy +++ b/tests/integration/update_cluster/public-jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -136,35 +136,6 @@ "Resource": [ "*" ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] } ] } diff --git a/tests/integration/update_cluster/public-jwks/kubernetes.tf b/tests/integration/update_cluster/public-jwks/kubernetes.tf index 5d2e0a6240..c01a09d30b 100644 --- a/tests/integration/update_cluster/public-jwks/kubernetes.tf +++ b/tests/integration/update_cluster/public-jwks/kubernetes.tf @@ -1,25 +1,35 @@ locals { - cluster_name = "minimal.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] - master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] - masters_role_arn = aws_iam_role.masters-minimal-example-com.arn - masters_role_name = aws_iam_role.masters-minimal-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] - node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] - nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn - nodes_role_name = aws_iam_role.nodes-minimal-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.minimal-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id - vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block - vpc_id = aws_vpc.minimal-example-com.id + cluster_name = "minimal.example.com" + kube-system-dns-controller_role_arn = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn + kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id } output "cluster_name" { value = "minimal.example.com" } +output "kube-system-dns-controller_role_arn" { + value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn +} + +output "kube-system-dns-controller_role_name" { + value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name +} + output "master_autoscaling_group_ids" { value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] } @@ -200,6 +210,12 @@ resource "aws_iam_openid_connect_provider" "minimal-example-com" { url = "https://api.minimal.example.com" } +resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" { + name = "dns-controller.kube-system.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy") + role = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name +} + resource "aws_iam_role_policy" "masters-minimal-example-com" { name = "masters.minimal.example.com" policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") @@ -212,6 +228,11 @@ resource "aws_iam_role_policy" "nodes-minimal-example-com" { role = aws_iam_role.nodes-minimal-example-com.name } +resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy") + name = "dns-controller.kube-system.sa.minimal.example.com" +} + resource "aws_iam_role" "masters-minimal-example-com" { assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") name = "masters.minimal.example.com" diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index ece3ac29d2..60eb5dfede 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -47,6 +47,8 @@ go_library( "//pkg/model/alimodel:go_default_library", "//pkg/model/awsmodel:go_default_library", "//pkg/model/components:go_default_library", + "//pkg/model/components/addonmanifests:go_default_library", + "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", "//pkg/model/components/etcdmanager:go_default_library", "//pkg/model/components/kubeapiserver:go_default_library", "//pkg/model/components/node-authorizer:go_default_library", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index aaf1487923..1575e24cac 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -408,6 +408,12 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { awsCloud := cloud.(awsup.AWSCloud) region = awsCloud.Region() + accountID, err := awsCloud.AccountID() + if err != nil { + return err + } + modelContext.AWSAccountID = accountID + if len(sshPublicKeys) == 0 && c.Cluster.Spec.SSHKeyName == nil { return fmt.Errorf("SSH public key must be specified when running with AWS (create with `kops create secret --name %s sshpublickey admin -i ~/.ssh/id_rsa.pub`)", cluster.ObjectMeta.Name) } diff --git a/upup/pkg/fi/cloudup/awstasks/iamrole.go b/upup/pkg/fi/cloudup/awstasks/iamrole.go index c9f0cb31a4..7fdac2059c 100644 --- a/upup/pkg/fi/cloudup/awstasks/iamrole.go +++ b/upup/pkg/fi/cloudup/awstasks/iamrole.go @@ -157,6 +157,7 @@ func (_ *IAMRole) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMRole) error response, err := t.Cloud.IAM().CreateRole(request) if err != nil { + klog.V(2).Infof("IAMRole policy: %s", policy) return fmt.Errorf("error creating IAMRole: %v", err) } diff --git a/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go b/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go index 368908555a..dc2ecd98a2 100644 --- a/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go +++ b/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go @@ -264,6 +264,7 @@ func (_ *IAMRolePolicy) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMRoleP _, err = t.Cloud.IAM().PutRolePolicy(request) if err != nil { + klog.V(2).Infof("PutRolePolicy RoleName=%s PolicyName=%s: %s", aws.StringValue(e.Role.Name), aws.StringValue(e.Name), policy) return fmt.Errorf("error creating/updating IAMRolePolicy: %v", err) } } diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 6d43b42ee8..32f65e01e9 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -40,6 +40,7 @@ import ( "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/aws/aws-sdk-go/service/route53" "github.com/aws/aws-sdk-go/service/route53/route53iface" + "github.com/aws/aws-sdk-go/service/sts" "k8s.io/klog/v2" v1 "k8s.io/api/core/v1" @@ -161,6 +162,9 @@ type AWSCloud interface { // FindClusterStatus gets the status of the cluster as it exists in AWS, inferred from volumes FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error) + + // AccountID returns the AWS account ID (typically a 12 digit number) we are deploying into + AccountID() (string, error) } type awsCloudImplementation struct { @@ -172,6 +176,7 @@ type awsCloudImplementation struct { autoscaling *autoscaling.AutoScaling route53 *route53.Route53 spotinst spotinst.Cloud + sts *sts.STS region string @@ -274,6 +279,14 @@ func NewAWSCloud(region string, tags map[string]string) (AWSCloud, error) { c.elbv2.Handlers.Send.PushFront(requestLogger) c.addHandlers(region, &c.elbv2.Handlers) + sess, err = session.NewSession(config) + if err != nil { + return c, err + } + c.sts = sts.New(sess, config) + c.sts.Handlers.Send.PushFront(requestLogger) + c.addHandlers(region, &c.sts.Handlers) + sess, err = session.NewSession(config) if err != nil { return c, err @@ -1639,3 +1652,20 @@ func describeInstanceType(c AWSCloud, instanceType string) (*ec2.InstanceTypeInf } return resp.InstanceTypes[0], nil } + +// AccountID returns the AWS account ID (typically a 12 digit number) we are deploying into +func (c *awsCloudImplementation) AccountID() (string, error) { + request := &sts.GetCallerIdentityInput{} + + response, err := c.sts.GetCallerIdentity(request) + if err != nil { + return "", fmt.Errorf("error geting AWS account ID: %v", err) + } + + account := aws.StringValue(response.Account) + if account == "" { + return "", fmt.Errorf("AWS account id was empty") + } + + return account, nil +} diff --git a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go index a5b1c37537..d44fa956f4 100644 --- a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go @@ -300,3 +300,8 @@ func (c *MockAWSCloud) DescribeInstanceType(instanceType string) (*ec2.InstanceT } return info, nil } + +// AccountID returns the AWS account ID (typically a 12 digit number) we are deploying into +func (c *MockAWSCloud) AccountID() (string, error) { + return "123456789012", nil +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 15deb3ae6c..76400caa4b 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -29,6 +29,9 @@ import ( "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/components/addonmanifests" + "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" + "k8s.io/kops/pkg/model/iam" "k8s.io/kops/pkg/templates" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" @@ -48,7 +51,11 @@ var _ fi.ModelBuilder = &BootstrapChannelBuilder{} // Build is responsible for adding the addons to the channel func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { - addons := b.buildAddons() + addons, err := b.buildAddons(c) + if err != nil { + return err + } + if err := addons.Verify(); err != nil { return err } @@ -71,7 +78,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("error reading manifest %s: %v", manifestPath, err) } - remapped, err := b.assetBuilder.RemapManifest(manifestBytes) + // Go through any transforms that are best expressed as code + remapped, err := addonmanifests.RemapAddonManifest(a, b.KopsModelContext, b.assetBuilder, manifestBytes) if err != nil { klog.Infof("invalid manifest: %s", string(manifestBytes)) return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err) @@ -157,7 +165,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { return nil } -func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { +func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*channelsapi.Addons, error) { addons := &channelsapi.Addons{} addons.Kind = "Addons" addons.ObjectMeta.Name = "bootstrap" @@ -461,6 +469,19 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { }) } } + + // Generate dns-controller ServiceAccount IAM permissions + if b.UseServiceAccountIAM() { + serviceAccountRoles := []iam.Subject{&dnscontroller.ServiceAccount{}} + for _, serviceAccountRole := range serviceAccountRoles { + iamModelBuilder := &model.IAMModelBuilder{KopsModelContext: b.KopsModelContext, Lifecycle: b.Lifecycle} + + err := iamModelBuilder.BuildServiceAccountRoleTasks(serviceAccountRole, c) + if err != nil { + return nil, err + } + } + } } if featureflag.EnableExternalDNS.Enabled() { @@ -1235,5 +1256,5 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { }) } - return addons + return addons, nil } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index 96d8810715..6e5877cd37 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -57,9 +57,9 @@ func TestBootstrapChannelBuilder_PublicJWKS(t *testing.T) { h.SetupMockAWS() - featureflag.ParseFlags("+PublicJWKS") + featureflag.ParseFlags("+PublicJWKS,+UseServiceAccountIAM") unsetFeatureFlag := func() { - featureflag.ParseFlags("-PublicJWKS") + featureflag.ParseFlags("-PublicJWKS,-UseServiceAccountIAM") } defer unsetFeatureFlag() runChannelBuilderTest(t, "public-jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-issuer-discovery.addons.k8s.io-k8s-1.16"}) diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml new file mode 100644 index 0000000000..01d2200b2e --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: anonymous-access.addons.k8s.io + name: anonymous:service-account-issuer-discovery + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:service-account-issuer-discovery +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml new file mode 100644 index 0000000000..91d03333db --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml @@ -0,0 +1,42 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + addons: + - manifest: s3://somebucket/example.yaml + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: events + iam: {} + kubernetesVersion: v1.14.6 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + additionalSans: + - proxy.api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml new file mode 100644 index 0000000000..255e5fa921 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml @@ -0,0 +1,128 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.19.0-alpha.2 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.19.0-alpha.2 + spec: + containers: + - command: + - /dns-controller + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --zone=*/* + - -v=2 + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + - name: AWS_ROLE_ARN + value: arn:aws:iam:::role/dns-controller.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: kope/dns-controller:1.19.0-alpha.2 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 + serviceAccount: dns-controller + tolerations: + - operator: Exists + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml new file mode 100644 index 0000000000..cd2920e40e --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -0,0 +1,175 @@ +apiVersion: v1 +data: + config.yaml: | + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"} +kind: ConfigMap +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.19.0-alpha.2 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.19.0-alpha.2 + spec: + containers: + - command: + - /kops-controller + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + image: kope/kops-controller:1.19.0-alpha.2 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-node-critical + serviceAccount: kops-controller + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + resourceNames: + - kops-controller-leader + resources: + - configmaps + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml new file mode 100644 index 0000000000..b618f7c380 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml @@ -0,0 +1,96 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + kubernetesVersion: '>=1.16.0-alpha.0' + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 + name: kops-controller.addons.k8s.io + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 1.19.0-alpha.2 + - id: k8s-1.16 + kubernetesVersion: '>=1.16.0-alpha.0' + manifest: anonymous-access.addons.k8s.io/k8s-1.16.yaml + manifestHash: d01bb2f3c12819e21bf0197624b95fb53dc0951a + name: anonymous-access.addons.k8s.io + selector: + k8s-addon: anonymous-access.addons.k8s.io + version: 1.19.0-alpha.3 + - manifest: core.addons.k8s.io/v1.4.0.yaml + manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90 + name: core.addons.k8s.io + selector: + k8s-addon: core.addons.k8s.io + version: 1.4.0 + - id: k8s-1.6 + kubernetesVersion: <1.12.0 + manifest: kube-dns.addons.k8s.io/k8s-1.6.yaml + manifestHash: 79dc1f02e5b03f6cfd06631bf26a9e4d3cb304f6 + name: kube-dns.addons.k8s.io + selector: + k8s-addon: kube-dns.addons.k8s.io + version: 1.15.13-kops.3 + - id: k8s-1.12 + kubernetesVersion: '>=1.12.0' + manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml + manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9 + name: kube-dns.addons.k8s.io + selector: + k8s-addon: kube-dns.addons.k8s.io + version: 1.15.13-kops.3 + - id: k8s-1.8 + manifest: rbac.addons.k8s.io/k8s-1.8.yaml + manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914 + name: rbac.addons.k8s.io + selector: + k8s-addon: rbac.addons.k8s.io + version: 1.8.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: v0.0.1 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 1.5.0 + - id: k8s-1.6 + kubernetesVersion: <1.12.0 + manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml + manifestHash: efb6c7b28be1e4cfe493f01b4a9f81a54f69f120 + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 1.19.0-alpha.2 + - id: k8s-1.12 + kubernetesVersion: '>=1.12.0' + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 5769bc1b5dd586e79338a42402dded07a3771634 + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 1.19.0-alpha.2 + - id: v1.15.0 + kubernetesVersion: '>=1.15.0' + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 00cf6e46e25b736b2da93c6025ce482474d83904 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.15.0 + - id: v1.7.0 + kubernetesVersion: <1.15.0 + manifest: storage-aws.addons.k8s.io/v1.7.0.yaml + manifestHash: 62705a596142e6cc283280e8aa973e51536994c5 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.15.0 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml index 4ff7647163..505c3aca88 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml @@ -34,6 +34,10 @@ spec: env: - name: KUBERNETES_SERVICE_HOST value: 127.0.0.1 + - name: AWS_ROLE_ARN + value: arn:aws:iam:::role/dns-controller.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token image: k8s.gcr.io/kops/dns-controller:1.19.0-alpha.3 name: dns-controller resources: @@ -42,14 +46,29 @@ spec: memory: 50Mi securityContext: runAsNonRoot: true + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true dnsPolicy: Default hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 serviceAccount: dns-controller tolerations: - operator: Exists + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml index 5246921401..be56243a62 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml @@ -73,7 +73,7 @@ spec: - id: k8s-1.12 kubernetesVersion: '>=1.12.0' manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml - manifestHash: 916cfc0d1bd7d7c6c80e85045d768c4de9178c62 + manifestHash: bcd96c6ebb616cb060250132b9bf802f59f7446b name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io