From 8c8455b8f4e58c1e53c6083dbb7e95c37c9ab3b7 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 12 Nov 2021 18:28:09 -0800 Subject: [PATCH] Upgrade external-dns to 0.10.1 for Kubernetes >= 1.19 --- cmd/kops/integration_test.go | 4 +- ...nimal.example.com-addons-bootstrap_content | 6 +- ...ternal-dns.addons.k8s.io-k8s-1.19_content} | 63 +++++++- .../update_cluster/external_dns/kubernetes.tf | 6 +- ...nimal.example.com-addons-bootstrap_content | 6 +- ...ternal-dns.addons.k8s.io-k8s-1.19_content} | 80 +++++++++- .../external_dns_irsa/kubernetes.tf | 6 +- upup/models/BUILD.bazel | 1 + .../k8s-1.19.yaml.template | 141 ++++++++++++++++++ .../bootstrapchannelbuilder.go | 9 +- 10 files changed, 292 insertions(+), 30 deletions(-) rename tests/integration/update_cluster/external_dns/data/{aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content => aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content} (66%) rename tests/integration/update_cluster/external_dns_irsa/data/{aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content => aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content} (57%) create mode 100644 upup/models/cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 3b2a12ede6..3f1e611ff4 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -538,7 +538,7 @@ func TestCCM(t *testing.T) { func TestExternalDNS(t *testing.T) { newIntegrationTest("minimal.example.com", "external_dns"). - withAddons("external-dns.addons.k8s.io-k8s-1.12"). + withAddons("external-dns.addons.k8s.io-k8s-1.19"). runTestTerraformAWS(t) newIntegrationTest("minimal.example.com", "external_dns").runTestCloudformation(t) } @@ -546,7 +546,7 @@ func TestExternalDNS(t *testing.T) { func TestExternalDNSIRSA(t *testing.T) { newIntegrationTest("minimal.example.com", "external_dns_irsa"). withOIDCDiscovery(). - withAddons("external-dns.addons.k8s.io-k8s-1.12"). + withAddons("external-dns.addons.k8s.io-k8s-1.19"). withServiceAccountRole("external-dns.kube-system", true). runTestTerraformAWS(t) } diff --git a/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 61a0b948d9..9f96fd2366 100644 --- a/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -38,9 +38,9 @@ spec: selector: k8s-addon: limit-range.addons.k8s.io version: 9.99.0 - - id: k8s-1.12 - manifest: external-dns.addons.k8s.io/k8s-1.12.yaml - manifestHash: c56e7a43d0d3e268d8d5d47fb014fd71309e933729ce186ded43624bb4270037 + - id: k8s-1.19 + manifest: external-dns.addons.k8s.io/k8s-1.19.yaml + manifestHash: 4ce9ba518a83cac3fae0cb1385085e18c581ee285eec08161a44422144e8a766 name: external-dns.addons.k8s.io selector: k8s-addon: external-dns.addons.k8s.io diff --git a/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content similarity index 66% rename from tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content rename to tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content index fe6781c556..374b01a066 100644 --- a/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/managed-by: kops k8s-addon: external-dns.addons.k8s.io k8s-app: external-dns - version: v0.9.0 + version: v0.10.1 name: external-dns namespace: kube-system spec: @@ -17,12 +17,10 @@ spec: k8s-app: external-dns template: metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" labels: k8s-addon: external-dns.addons.k8s.io k8s-app: external-dns - version: v0.9.0 + version: v0.10.1 spec: containers: - args: @@ -40,19 +38,49 @@ spec: value: 127.0.0.1 - name: KUBERNETES_SERVICE_PORT value: "443" - image: k8s.gcr.io/external-dns/external-dns:v0.9.0 - imagePullPolicy: Always + image: k8s.gcr.io/external-dns/external-dns:v0.10.1 + livenessProbe: + failureThreshold: 2 + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: external-dns + ports: + - containerPort: 7979 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: requests: cpu: 50m memory: 50Mi + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 dnsPolicy: Default hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical - serviceAccount: external-dns + securityContext: + fsGroup: 65534 + serviceAccountName: external-dns tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master @@ -61,6 +89,27 @@ spec: --- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: external-dns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: external-dns.addons.k8s.io + name: kops:external-dns +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + k8s-app: external-dns + type: ClusterIP + +--- + apiVersion: v1 kind: ServiceAccount metadata: diff --git a/tests/integration/update_cluster/external_dns/kubernetes.tf b/tests/integration/update_cluster/external_dns/kubernetes.tf index b04d362d82..ad1413d60a 100644 --- a/tests/integration/update_cluster/external_dns/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns/kubernetes.tf @@ -567,10 +567,10 @@ resource "aws_s3_bucket_object" "minimal-example-com-addons-coredns-addons-k8s-i server_side_encryption = "AES256" } -resource "aws_s3_bucket_object" "minimal-example-com-addons-external-dns-addons-k8s-io-k8s-1-12" { +resource "aws_s3_bucket_object" "minimal-example-com-addons-external-dns-addons-k8s-io-k8s-1-19" { bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content") - key = "clusters.example.com/minimal.example.com/addons/external-dns.addons.k8s.io/k8s-1.12.yaml" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content") + key = "clusters.example.com/minimal.example.com/addons/external-dns.addons.k8s.io/k8s-1.19.yaml" provider = aws.files server_side_encryption = "AES256" } diff --git a/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 61a0b948d9..b7ca0b8c8b 100644 --- a/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -38,9 +38,9 @@ spec: selector: k8s-addon: limit-range.addons.k8s.io version: 9.99.0 - - id: k8s-1.12 - manifest: external-dns.addons.k8s.io/k8s-1.12.yaml - manifestHash: c56e7a43d0d3e268d8d5d47fb014fd71309e933729ce186ded43624bb4270037 + - id: k8s-1.19 + manifest: external-dns.addons.k8s.io/k8s-1.19.yaml + manifestHash: b79531101b4ef4c387bbf6fc7cc0f712496b0811d7c0060413715892cd3d4b92 name: external-dns.addons.k8s.io selector: k8s-addon: external-dns.addons.k8s.io diff --git a/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content similarity index 57% rename from tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content rename to tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content index fe6781c556..e40ec39358 100644 --- a/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/managed-by: kops k8s-addon: external-dns.addons.k8s.io k8s-app: external-dns - version: v0.9.0 + version: v0.10.1 name: external-dns namespace: kube-system spec: @@ -17,12 +17,10 @@ spec: k8s-app: external-dns template: metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" labels: k8s-addon: external-dns.addons.k8s.io k8s-app: external-dns - version: v0.9.0 + version: v0.10.1 spec: containers: - args: @@ -40,24 +38,92 @@ spec: value: 127.0.0.1 - name: KUBERNETES_SERVICE_PORT value: "443" - image: k8s.gcr.io/external-dns/external-dns:v0.9.0 - imagePullPolicy: Always + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/external-dns.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: k8s.gcr.io/external-dns/external-dns:v0.10.1 + livenessProbe: + failureThreshold: 2 + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 name: external-dns + ports: + - containerPort: 7979 + name: http + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: requests: cpu: 50m memory: 50Mi + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true dnsPolicy: Default hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical - serviceAccount: external-dns + securityContext: + fsGroup: 65534 + serviceAccountName: external-dns tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master - effect: NoSchedule key: node.kubernetes.io/not-ready + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + +--- + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: external-dns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: external-dns.addons.k8s.io + name: kops:external-dns +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + k8s-app: external-dns + type: ClusterIP --- diff --git a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf index bed8a3e03f..3590156056 100644 --- a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf @@ -630,10 +630,10 @@ resource "aws_s3_bucket_object" "minimal-example-com-addons-coredns-addons-k8s-i server_side_encryption = "AES256" } -resource "aws_s3_bucket_object" "minimal-example-com-addons-external-dns-addons-k8s-io-k8s-1-12" { +resource "aws_s3_bucket_object" "minimal-example-com-addons-external-dns-addons-k8s-io-k8s-1-19" { bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.12_content") - key = "clusters.example.com/minimal.example.com/addons/external-dns.addons.k8s.io/k8s-1.12.yaml" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-external-dns.addons.k8s.io-k8s-1.19_content") + key = "clusters.example.com/minimal.example.com/addons/external-dns.addons.k8s.io/k8s-1.19.yaml" provider = aws.files server_side_encryption = "AES256" } diff --git a/upup/models/BUILD.bazel b/upup/models/BUILD.bazel index 902fc15024..78352f6ea9 100644 --- a/upup/models/BUILD.bazel +++ b/upup/models/BUILD.bazel @@ -52,6 +52,7 @@ go_library( "cloudup/resources/addons/node-problem-detector.addons.k8s.io/k8s-1.17.yaml.template", "cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml", "cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.22.yaml.template", + "cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template", ], importpath = "k8s.io/kops/upup/models", visibility = ["//visibility:public"], diff --git a/upup/models/cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template new file mode 100644 index 0000000000..6801ab7ac0 --- /dev/null +++ b/upup/models/cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template @@ -0,0 +1,141 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: external-dns + namespace: kube-system + labels: + k8s-addon: external-dns.addons.k8s.io + k8s-app: external-dns + version: v0.10.1 +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: external-dns + template: + metadata: + labels: + k8s-addon: external-dns.addons.k8s.io + k8s-app: external-dns + version: v0.10.1 + spec: + priorityClassName: system-cluster-critical + serviceAccountName: external-dns + securityContext: + fsGroup: 65534 + tolerations: + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + - key: "node.kubernetes.io/not-ready" + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" + dnsPolicy: Default # Don't use cluster DNS (we are likely running before kube-dns) + hostNetwork: true + containers: + - name: external-dns + securityContext: + runAsNonRoot: true + runAsUser: 65534 + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + image: k8s.gcr.io/external-dns/external-dns:v0.10.1 + args: +{{ range $arg := ExternalDnsArgv }} + - "{{ $arg }}" +{{ end }} + env: + - name: KUBERNETES_SERVICE_HOST + value: "127.0.0.1" + - name: KUBERNETES_SERVICE_PORT + value: "443" + ports: + - name: http + protocol: TCP + containerPort: 7979 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 2 + successThreshold: 1 + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + resources: + requests: + cpu: 50m + memory: 50Mi +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-addon: external-dns.addons.k8s.io + name: kops:external-dns +spec: + type: ClusterIP + selector: + k8s-app: external-dns + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-dns + namespace: kube-system + labels: + k8s-addon: external-dns.addons.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: external-dns.addons.k8s.io + name: kops:external-dns +rules: +- apiGroups: [""] + resources: ["services","endpoints","pods"] + verbs: ["get","watch","list"] +- apiGroups: ["extensions","networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get","watch","list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: external-dns.addons.k8s.io + name: kops:external-dns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:external-dns +subjects: +- kind: ServiceAccount + name: external-dns + namespace: kube-system diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index df44f1d0c8..e28c31b2f5 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -490,8 +490,13 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon key := "external-dns.addons.k8s.io" { - location := key + "/k8s-1.12.yaml" - id := "k8s-1.12" + location := key + "/k8s-1.19.yaml" + id := "k8s-1.19" + + if b.IsKubernetesLT("1.19") { + location = key + "/k8s-1.12.yaml" + id = "k8s-1.12" + } addons.Add(&channelsapi.AddonSpec{ Name: fi.String(key),