From 28bd45a8fa7577ec9f245a6c476414f8c7fcc76d Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Mon, 19 Jul 2021 10:21:40 +0200 Subject: [PATCH] Add irsa support for nth --- .../components/addonmanifests/BUILD.bazel | 1 + .../nodeterminationhandler/BUILD.bazel | 12 +++++ .../nodeterminationhandler/iam.go | 48 +++++++++++++++++++ pkg/model/components/addonmanifests/remap.go | 3 ++ pkg/model/iam/iam_builder.go | 20 +++++--- .../nth_sqs_resources/cloudformation.json | 2 +- ...masters.nthsqsresources.example.com_policy | 2 +- ...urces.example.com-addons-bootstrap_content | 2 +- ...e-termination-handler.aws-k8s-1.11_content | 2 +- .../k8s-1.11.yaml.template | 4 +- .../bootstrapchannelbuilder/BUILD.bazel | 1 + .../bootstrapchannelbuilder.go | 5 ++ 12 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 pkg/model/components/addonmanifests/nodeterminationhandler/BUILD.bazel create mode 100644 pkg/model/components/addonmanifests/nodeterminationhandler/iam.go diff --git a/pkg/model/components/addonmanifests/BUILD.bazel b/pkg/model/components/addonmanifests/BUILD.bazel index ef7af739a9..82a5b03dbe 100644 --- a/pkg/model/components/addonmanifests/BUILD.bazel +++ b/pkg/model/components/addonmanifests/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/model/components/addonmanifests/awsloadbalancercontroller:go_default_library", "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library", "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", + "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library", "//pkg/model/iam:go_default_library", "//upup/pkg/fi:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/model/components/addonmanifests/nodeterminationhandler/BUILD.bazel b/pkg/model/components/addonmanifests/nodeterminationhandler/BUILD.bazel new file mode 100644 index 0000000000..dcc69cdd5d --- /dev/null +++ b/pkg/model/components/addonmanifests/nodeterminationhandler/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["iam.go"], + importpath = "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler", + visibility = ["//visibility:public"], + deps = [ + "//pkg/model/iam:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) diff --git a/pkg/model/components/addonmanifests/nodeterminationhandler/iam.go b/pkg/model/components/addonmanifests/nodeterminationhandler/iam.go new file mode 100644 index 0000000000..7a1997401b --- /dev/null +++ b/pkg/model/components/addonmanifests/nodeterminationhandler/iam.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeterminationhandler + +import ( + "k8s.io/apimachinery/pkg/types" + "k8s.io/kops/pkg/model/iam" +) + +// ServiceAccount represents the service-account used by the dns-controller. +// It implements iam.Subject to get AWS IAM permissions. +type ServiceAccount struct { +} + +var _ iam.Subject = &ServiceAccount{} + +// BuildAWSPolicy generates a custom policy for a ServiceAccount IAM role. +func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, error) { + + clusterName := b.Cluster.ObjectMeta.Name + p := iam.NewPolicy(clusterName) + + iam.AddNodeTerminationHandlerSQSPermissions(p) + + return p, nil +} + +// ServiceAccount returns the kubernetes service account used. +func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{ + Namespace: "kube-system", + Name: "aws-node-termination-handler", + }, true +} diff --git a/pkg/model/components/addonmanifests/remap.go b/pkg/model/components/addonmanifests/remap.go index 4d9f423d7f..571900b410 100644 --- a/pkg/model/components/addonmanifests/remap.go +++ b/pkg/model/components/addonmanifests/remap.go @@ -32,6 +32,7 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests/awsloadbalancercontroller" "k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler" "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" + "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/upup/pkg/fi" ) @@ -123,6 +124,8 @@ func getWellknownServiceAccount(name string) iam.Subject { return &clusterautoscaler.ServiceAccount{} case "ebs-csi-controller-sa": return &awsebscsidriver.ServiceAccount{} + case "aws-node-termination-handler": + return &nodeterminationhandler.ServiceAccount{} default: return nil } diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 4f931ff9a0..cc22c8205d 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -339,6 +339,11 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { AddAWSLoadbalancerControllerPermissions(p) } AddClusterAutoscalerPermissions(p) + + nth := b.Cluster.Spec.NodeTerminationHandler + if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) { + AddNodeTerminationHandlerSQSPermissions(p) + } } if b.Cluster.Spec.IAM.AllowContainerRegistry { @@ -361,11 +366,6 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { addCalicoSrcDstCheckPermissions(p) } - nth := b.Cluster.Spec.NodeTerminationHandler - if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) { - addNodeTerminationHandlerSQSPermissions(p) - } - if b.Cluster.Spec.SnapshotController != nil && fi.BoolValue(b.Cluster.Spec.SnapshotController.Enabled) { addSnapshotPersmissions(p) } @@ -1170,11 +1170,17 @@ func addAmazonVPCCNIPermissions(p *Policy, iamPrefix string) { ) } -func addNodeTerminationHandlerSQSPermissions(p *Policy) { +func AddNodeTerminationHandlerSQSPermissions(p *Policy) { p.unconditionalAction.Insert( - "autoscaling:CompleteLifecycleAction", "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + // SQS permissions do not support conditions. "sqs:DeleteMessage", "sqs:ReceiveMessage", ) + p.clusterTaggedAction.Insert( + "autoscaling:CompleteLifecycleAction", + ) + } diff --git a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json index 5acc58f2e1..634385ba87 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json +++ b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json @@ -1234,7 +1234,6 @@ }, { "Action": [ - "autoscaling:CompleteLifecycleAction", "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", @@ -1269,6 +1268,7 @@ }, { "Action": [ + "autoscaling:CompleteLifecycleAction", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup", "ec2:AttachVolume", diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy index 290434874f..2cbd3c3f46 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy @@ -172,7 +172,6 @@ }, { "Action": [ - "autoscaling:CompleteLifecycleAction", "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", @@ -207,6 +206,7 @@ }, { "Action": [ + "autoscaling:CompleteLifecycleAction", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup", "ec2:AttachVolume", diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-bootstrap_content b/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-bootstrap_content index 4125365ea6..f5635a8bf1 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: k8s-addon: dns-controller.addons.k8s.io - id: k8s-1.11 manifest: node-termination-handler.aws/k8s-1.11.yaml - manifestHash: 57274d900239a8ba937e5887c4fa8b276165f7f0 + manifestHash: 4959667b24bdb70ded01a59a2c502ef197b0d8f4 name: node-termination-handler.aws selector: k8s-addon: node-termination-handler.aws diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-node-termination-handler.aws-k8s-1.11_content b/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-node-termination-handler.aws-k8s-1.11_content index cf1bd80f36..9a9418c04e 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-node-termination-handler.aws-k8s-1.11_content +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-node-termination-handler.aws-k8s-1.11_content @@ -214,7 +214,7 @@ spec: hostNetwork: false nodeSelector: node-role.kubernetes.io/master: "" - priorityClassName: system-node-critical + priorityClassName: system-cluster-critical securityContext: fsGroup: 1000 serviceAccountName: aws-node-termination-handler diff --git a/upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template b/upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template index b79bfba870..5890982129 100644 --- a/upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template +++ b/upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template @@ -94,7 +94,7 @@ spec: k8s-app: aws-node-termination-handler kubernetes.io/os: linux spec: - priorityClassName: "system-node-critical" + priorityClassName: "system-cluster-critical" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -200,10 +200,12 @@ spec: requests: cpu: {{ .CPURequest }} memory: {{ .MemoryRequest }} + {{ if not UseServiceAccountIAM }} nodeSelector: node-role.kubernetes.io/master: "" tolerations: - operator: Exists + {{ end }} {{ else }} --- # Source: aws-node-termination-handler/templates/daemonset.linux.yaml diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel index 8646783fc4..eb409b3430 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//pkg/model/components/addonmanifests/awsloadbalancercontroller:go_default_library", "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library", "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", + "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library", "//pkg/model/iam:go_default_library", "//pkg/templates:go_default_library", "//pkg/wellknownoperators:go_default_library", diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index f5477124ad..70807ba6e2 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -33,6 +33,7 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests/awsloadbalancercontroller" "k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler" "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" + "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/pkg/templates" "k8s.io/kops/pkg/wellknownoperators" @@ -559,6 +560,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann Id: id, }) } + + if b.UseServiceAccountIAM() { + serviceAccountRoles = append(serviceAccountRoles, &nodeterminationhandler.ServiceAccount{}) + } } npd := b.Cluster.Spec.NodeProblemDetector