Merge pull request #8634 from geojaz/gce_metadata

Node metadata-concealment in GCE
2020-03-14 13:00:34 -07:00 · 2020-03-14 13:00:34 -07:00 · cc52fa3812
parent 7d030ae459 d39006e2b5
commit cc52fa3812
7 changed files with 153 additions and 5 deletions
--- a/cmd/kops/create_cluster.go
+++ b/cmd/kops/create_cluster.go
@ -231,9 +231,9 @@ var (
 		--master-zones $ZONES \
 		--node-count 3 \
 		--yes
-		  
-	# Generate a cluster spec to apply later. 
-	# Run the following, then: kops create -f filename.yamlh 
+
+	# Generate a cluster spec to apply later.
+	# Run the following, then: kops create -f filename.yamlh
 	kops create cluster --name=kubernetes-cluster.example.com \
 		--state=s3://kops-state-1234 \
 		--zones=eu-west-1a \
@ -1293,6 +1293,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 			return err
 		}
 		fullGroup.AddInstanceGroupNodeLabel()
+		if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderGCE {
+			fullGroup.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
+		}
 		fullInstanceGroups = append(fullInstanceGroups, fullGroup)
 	}

--- a/cmd/kops/create_ig.go
+++ b/cmd/kops/create_ig.go
@ -163,6 +163,10 @@ func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string,
 	}

 	ig.AddInstanceGroupNodeLabel()
+	if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderGCE {
+		fmt.Println("detected a GCE cluster; labeling nodes to receive metadata-proxy.")
+		ig.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
+	}

 	if options.DryRun {

--- a/docs/releases/1.18-NOTES.md
+++ b/docs/releases/1.18-NOTES.md
@ -10,6 +10,8 @@

 * Cilium CNI can now use AWS networking natively through the AWS ENI IPAM mode. Kops can also run a Kubernetes cluster entirely without kube-proxy using Cilium's BPF NodePort implementation

+* New clusters in GCE are configured to run the [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) by default. The proxy runs as a DaemonSet and lands on nodes with the nodeLabel `cloud.google.com/metadata-proxy-ready: "true"`. If you want to enable metadata-proxy on an existing cluster/instance group, add that nodeLabel to your instancegroup specs (`kops edit ig ...`) and run `kops update cluster`. When the changes are applied, the proxy will roll out to those targeted nodes.
+
 # Breaking changes

 * Terraform users on AWS may need to rename some resources in their state file in order to prepare for Terraform 0.12 support. See Required Actions below.
@ -30,7 +32,7 @@
  Terraform 0.12 [no longer supports resource names starting with digits](https://www.terraform.io/upgrade-guides/0-12.html#pre-upgrade-checklist). In Kops, both the default route and additional VPC CIDR associations are affected. See [#7957](https://github.com/kubernetes/kops/pull/7957) for more information.
  * The default route was named `aws_route.0-0-0-0--0` and will now be named `aws_route.route-0-0-0-0--0`.
  * Additional CIDR blocks associated with a VPC were similarly named the hyphenated CIDR block with two hyphens for the `/`, for example `aws_vpc_ipv4_cidr_block_association.10-1-0-0--16`. These will now be prefixed with `cidr-`, for example `aws_vpc_ipv4_cidr_block_association.cidr-10-1-0-0--16`.
-  
+
  To prevent downtime, follow these steps with the new version of Kops:
  ```
  kops update cluster --target terraform ...
@ -55,7 +57,7 @@
      featureGates:
        PodPriority: "true"
  ```
- 
+
 * If a custom Kops build was used on a cluster, a kops-controller Deployment may have been created that should get deleted.
  Run `kubectl -n kube-system delete deployment kops-controller` after upgrading to Kops 1.16.0-beta.1 or later.

--- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml
+++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml
@ -73,6 +73,7 @@ spec:
  maxSize: 1
  minSize: 1
  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
    kops.k8s.io/instancegroup: master-us-test1-a
  role: Master
  subnets:
@ -95,6 +96,7 @@ spec:
  maxSize: 1
  minSize: 1
  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
    kops.k8s.io/instancegroup: master-us-test1-b
  role: Master
  subnets:
@ -117,6 +119,7 @@ spec:
  maxSize: 1
  minSize: 1
  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
    kops.k8s.io/instancegroup: master-us-test1-c
  role: Master
  subnets:
@ -139,6 +142,7 @@ spec:
  maxSize: 2
  minSize: 2
  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
    kops.k8s.io/instancegroup: nodes
  role: Node
  subnets:
--- a/upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml
+++ b/upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml
@ -0,0 +1,10 @@
+kind: Addons
+metadata:
+  name: metadata-proxy
+spec:
+  addons:
+  - version: 0.1.12
+    selector:
+      k8s-addon: metadata-proxy.addons.k8s.io
+    manifest: v0.12.yaml
+
--- a/upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml
+++ b/upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml
@ -0,0 +1,105 @@
+# Borrowed from https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metadata-proxy
+  namespace: kube-system
+  labels:
+    k8s-app: metadata-proxy
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: metadata-proxy-v0.12
+  namespace: kube-system
+  labels:
+    k8s-app: metadata-proxy
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    version: v0.12
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metadata-proxy
+      version: v0.12
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: metadata-proxy
+        kubernetes.io/cluster-service: "true"
+        version: v0.12
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: metadata-proxy
+      hostNetwork: true
+      dnsPolicy: Default
+      tolerations:
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
+      hostNetwork: true
+      initContainers:
+      - name: update-ipdtables
+        securityContext:
+          privileged: true
+        image: gcr.io/google_containers/k8s-custom-iptables:1.0
+        imagePullPolicy: Always
+        command: [ "/bin/sh", "-c", "/sbin/iptables -t nat -I PREROUTING -p tcp --dport 80 -d 169.254.169.254 -j DNAT --to-destination 127.0.0.1:988" ]
+        volumeMounts:
+        - name: host
+          mountPath: /host
+      volumes:
+      - name: host
+        hostPath:
+          path: /
+          type: Directory
+      containers:
+      - name: metadata-proxy
+        image: k8s.gcr.io/metadata-proxy:v0.1.12
+        securityContext:
+          privileged: true
+        # Request and limit resources to get guaranteed QoS.
+        resources:
+          requests:
+            memory: "25Mi"
+            cpu: "30m"
+          limits:
+            memory: "25Mi"
+            cpu: "30m"
+      # BEGIN_PROMETHEUS_TO_SD
+      - name: prometheus-to-sd-exporter
+        image: k8s.gcr.io/prometheus-to-sd:v0.5.0
+        # Request and limit resources to get guaranteed QoS.
+        resources:
+          requests:
+            memory: "20Mi"
+            cpu: "2m"
+          limits:
+            memory: "20Mi"
+            cpu: "2m"
+        command:
+          - /monitor
+          - --stackdriver-prefix=custom.googleapis.com/addons
+          - --source=metadata_proxy:http://127.0.0.1:989?whitelisted=request_count
+          - --pod-id=$(POD_NAME)
+          - --namespace-id=$(POD_NAMESPACE)
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+      # END_PROMETHEUS_TO_SD
+      nodeSelector:
+        cloud.google.com/metadata-proxy-ready: "true"
+        beta.kubernetes.io/os: linux
+      terminationGracePeriodSeconds: 30
--- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
+++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
@ -553,6 +553,26 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 		}
 	}

+	// The metadata-proxy daemonset conceals node metadata endpoints in GCE.
+	// It will land on nodes labeled cloud.google.com/metadata-proxy-ready=true
+	if kops.CloudProviderID(b.cluster.Spec.CloudProvider) == kops.CloudProviderGCE {
+		key := "metadata-proxy.addons.k8s.io"
+		version := "0.1.12"
+
+		{
+			id := "v0.1.12"
+			location := key + "/" + id + ".yaml"
+
+			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				Name:     fi.String(key),
+				Version:  fi.String(version),
+				Selector: map[string]string{"k8s-addon": key},
+				Manifest: fi.String(location),
+				Id:       id,
+			})
+		}
+	}
+
 	// The role.kubernetes.io/networking is used to label anything related to a networking addin,
 	// so that if we switch networking plugins (e.g. calico -> weave or vice-versa), we'll replace the
 	// old networking plugin, and there won't be old pods "floating around".