From 49e816cf92c2c82973dc33bac458753c3f423d04 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 07:58:50 +0200 Subject: [PATCH 1/8] Amazon VPC CNI - rearrange manifest resources --- .../k8s-1.16.yaml.template | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 8db5368815..00dfb158a0 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -1,5 +1,18 @@ # Vendored from https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.7/config/v1.7/aws-k8s-cni.yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aws-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node +subjects: +- kind: ServiceAccount + name: aws-node + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -25,28 +38,22 @@ rules: resources: - daemonsets verbs: ["list", "watch"] - --- -apiVersion: v1 -kind: ServiceAccount +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: aws-node - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aws-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node -subjects: -- kind: ServiceAccount - name: aws-node - namespace: kube-system - + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + versions: + - name: v1alpha1 + served: true + storage: true + names: + plural: eniconfigs + singular: eniconfig + kind: ENIConfig --- apiVersion: apps/v1 kind: DaemonSet @@ -207,20 +214,9 @@ spec: path: /var/run/aws-node type: DirectoryOrCreate name: run-dir - --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition +apiVersion: v1 +kind: ServiceAccount metadata: - name: eniconfigs.crd.k8s.amazonaws.com -spec: - scope: Cluster - group: crd.k8s.amazonaws.com - versions: - - name: v1alpha1 - served: true - storage: true - names: - plural: eniconfigs - singular: eniconfig - kind: ENIConfig + name: aws-node + namespace: kube-system From 421fa24d3bb5d852736a926986cb60d33226f49b Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:13:30 +0200 Subject: [PATCH 2/8] Amazon VPC CNI - rearrange ClusterRole sections --- .../k8s-1.16.yaml.template | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 00dfb158a0..e2dde568f3 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -25,19 +25,32 @@ rules: - "*" verbs: - "*" -- apiGroups: [""] +- apiGroups: + - "" resources: - pods - namespaces - verbs: ["list", "watch", "get"] -- apiGroups: [""] + verbs: + - list + - watch + - get +- apiGroups: + - "" resources: - nodes - verbs: ["list", "watch", "get", "update"] -- apiGroups: ["extensions", "apps"] + verbs: + - list + - watch + - get + - update +- apiGroups: + - extensions + - apps resources: - daemonsets - verbs: ["list", "watch"] + verbs: + - list + - watch --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition From 14b33ae34ce9226bc45430e3ea9593c8a6e514c0 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:16:31 +0200 Subject: [PATCH 3/8] Amazon VPC CNI - update ClusterRole --- .../k8s-1.16.yaml.template | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index e2dde568f3..2f96a757f3 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -22,9 +22,11 @@ rules: - apiGroups: - crd.k8s.amazonaws.com resources: - - "*" + - eniconfigs verbs: - - "*" + - get + - list + - watch - apiGroups: - "" resources: @@ -47,7 +49,7 @@ rules: - extensions - apps resources: - - daemonsets + - "*" verbs: - list - watch From e6459e104ff7b2b44b755c7d291edd780cd76688 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:17:39 +0200 Subject: [PATCH 4/8] Amazon VPC CNI - rearrange ClusterRole --- .../k8s-1.16.yaml.template | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 2f96a757f3..1dbf0fe4fe 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -59,16 +59,16 @@ kind: CustomResourceDefinition metadata: name: eniconfigs.crd.k8s.amazonaws.com spec: - scope: Cluster group: crd.k8s.amazonaws.com + names: + kind: ENIConfig + plural: eniconfigs + singular: eniconfig + scope: Cluster versions: - name: v1alpha1 served: true storage: true - names: - plural: eniconfigs - singular: eniconfig - kind: ENIConfig --- apiVersion: apps/v1 kind: DaemonSet From 46122cd12b2c823130d12cf050d67c62633f9e71 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:38:54 +0200 Subject: [PATCH 5/8] Amazon VPC CNI - rearrange DaemonSet --- .../k8s-1.16.yaml.template | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 1dbf0fe4fe..8b06d38d07 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -80,10 +80,6 @@ metadata: labels: k8s-app: aws-node spec: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: "10%" selector: matchLabels: k8s-app: aws-node @@ -92,7 +88,6 @@ spec: labels: k8s-app: aws-node spec: - priorityClassName: system-node-critical affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -111,28 +106,8 @@ spec: operator: NotIn values: - fargate - serviceAccountName: aws-node - hostNetwork: true - tolerations: - - operator: Exists containers: - - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" - imagePullPolicy: Always - ports: - - containerPort: 61678 - name: metrics - name: aws-node - readinessProbe: - exec: - command: ["/app/grpc-health-probe", "-addr=:50051"] - initialDelaySeconds: 1 - livenessProbe: - exec: - command: ["/app/grpc-health-probe", "-addr=:50051"] - initialDelaySeconds: 60 - env: - - name: CLUSTER_NAME - value: {{ ClusterName }} + - env: - name: ADDITIONAL_ENI_TAGS value: '{}' - name: AWS_VPC_CNI_NODE_PORT_SUPPORT @@ -151,6 +126,8 @@ spec: value: /host/var/log/aws-routed-eni/ipamd.log - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT value: prng + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni - name: AWS_VPC_K8S_PLUGIN_LOG_FILE value: /var/log/aws-routed-eni/plugin.log - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL @@ -161,18 +138,36 @@ spec: value: "false" - name: ENABLE_POD_ENI value: "false" - - name: AWS_VPC_K8S_CNI_VETHPREFIX - value: eni - name: MY_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: WARM_ENI_TARGET value: "1" + - name: CLUSTER_NAME + value: {{ ClusterName }} {{- range .Networking.AmazonVPC.Env }} - name: {{ .Name }} value: "{{ .Value }}" {{- end }} + image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" + imagePullPolicy: Always + livenessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + initialDelaySeconds: 60 + name: aws-node + ports: + - containerPort: 61678 + name: metrics + readinessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + initialDelaySeconds: 1 resources: requests: cpu: 10m @@ -193,6 +188,7 @@ spec: name: dockershim - mountPath: /run/xtables.lock name: xtables-lock + hostNetwork: true initContainers: - env: - name: DISABLE_TCP_EARLY_DEMUX @@ -208,6 +204,10 @@ spec: volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir + priorityClassName: system-node-critical + serviceAccountName: aws-node + tolerations: + - operator: Exists volumes: - hostPath: path: /opt/cni/bin @@ -229,6 +229,10 @@ spec: path: /var/run/aws-node type: DirectoryOrCreate name: run-dir + updateStrategy: + rollingUpdate: + maxUnavailable: "10%" + type: RollingUpdate --- apiVersion: v1 kind: ServiceAccount From a36fa3f4e1e50f437098348560a8a8d55511fb66 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:42:21 +0200 Subject: [PATCH 6/8] Amazon VPC CNI - update DaemonSet --- .../networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 8b06d38d07..1ff3d2ff20 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -77,8 +77,6 @@ metadata: k8s-app: aws-node name: aws-node namespace: kube-system - labels: - k8s-app: aws-node spec: selector: matchLabels: @@ -196,16 +194,14 @@ spec: image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5 imagePullPolicy: Always name: aws-vpc-cni-init - resources: {} securityContext: privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir priorityClassName: system-node-critical serviceAccountName: aws-node + terminationGracePeriodSeconds: 10 tolerations: - operator: Exists volumes: From 56fda317d442f5fb4d7631375593105f51df1644 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:48:19 +0200 Subject: [PATCH 7/8] Amazon VPC CNI - quote all keys and values --- .../k8s-1.16.yaml.template | 451 +++++++++--------- 1 file changed, 226 insertions(+), 225 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template index 1ff3d2ff20..f695ae993e 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -1,237 +1,238 @@ # Vendored from https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.7/config/v1.7/aws-k8s-cni.yaml --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aws-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node -subjects: -- kind: ServiceAccount - name: aws-node - namespace: kube-system +"apiVersion": "rbac.authorization.k8s.io/v1" +"kind": "ClusterRoleBinding" +"metadata": + "name": "aws-node" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "ClusterRole" + "name": "aws-node" +"subjects": +- "kind": "ServiceAccount" + "name": "aws-node" + "namespace": "kube-system" --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aws-node -rules: -- apiGroups: - - crd.k8s.amazonaws.com - resources: - - eniconfigs - verbs: - - get - - list - - watch -- apiGroups: +"apiVersion": "rbac.authorization.k8s.io/v1" +"kind": "ClusterRole" +"metadata": + "name": "aws-node" +"rules": +- "apiGroups": + - "crd.k8s.amazonaws.com" + "resources": + - "eniconfigs" + "verbs": + - "get" + - "list" + - "watch" +- "apiGroups": - "" - resources: - - pods - - namespaces - verbs: - - list - - watch - - get -- apiGroups: + "resources": + - "pods" + - "namespaces" + "verbs": + - "list" + - "watch" + - "get" +- "apiGroups": - "" - resources: - - nodes - verbs: - - list - - watch - - get - - update -- apiGroups: - - extensions - - apps - resources: + "resources": + - "nodes" + "verbs": + - "list" + - "watch" + - "get" + - "update" +- "apiGroups": + - "extensions" + - "apps" + "resources": - "*" - verbs: - - list - - watch + "verbs": + - "list" + - "watch" --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: eniconfigs.crd.k8s.amazonaws.com -spec: - group: crd.k8s.amazonaws.com - names: - kind: ENIConfig - plural: eniconfigs - singular: eniconfig - scope: Cluster - versions: - - name: v1alpha1 - served: true - storage: true +"apiVersion": "apiextensions.k8s.io/v1beta1" +"kind": "CustomResourceDefinition" +"metadata": + "name": "eniconfigs.crd.k8s.amazonaws.com" +"spec": + "group": "crd.k8s.amazonaws.com" + "names": + "kind": "ENIConfig" + "plural": "eniconfigs" + "singular": "eniconfig" + "scope": "Cluster" + "versions": + - "name": "v1alpha1" + "served": true + "storage": true --- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: aws-node - name: aws-node - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: aws-node - template: - metadata: - labels: - k8s-app: aws-node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/os" - operator: In - values: - - linux - - key: "kubernetes.io/arch" - operator: In - values: - - amd64 - - arm64 - - key: "eks.amazonaws.com/compute-type" - operator: NotIn - values: - - fargate - containers: - - env: - - name: ADDITIONAL_ENI_TAGS - value: '{}' - - name: AWS_VPC_CNI_NODE_PORT_SUPPORT - value: "true" - - name: AWS_VPC_ENI_MTU - value: "9001" - - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER - value: "false" - - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG - value: "false" - - name: AWS_VPC_K8S_CNI_EXTERNALSNAT - value: "false" - - name: AWS_VPC_K8S_CNI_LOGLEVEL - value: DEBUG - - name: AWS_VPC_K8S_CNI_LOG_FILE - value: /host/var/log/aws-routed-eni/ipamd.log - - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT - value: prng - - name: AWS_VPC_K8S_CNI_VETHPREFIX - value: eni - - name: AWS_VPC_K8S_PLUGIN_LOG_FILE - value: /var/log/aws-routed-eni/plugin.log - - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL - value: DEBUG - - name: DISABLE_INTROSPECTION - value: "false" - - name: DISABLE_METRICS - value: "false" - - name: ENABLE_POD_ENI - value: "false" - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: WARM_ENI_TARGET - value: "1" - - name: CLUSTER_NAME - value: {{ ClusterName }} +"apiVersion": "apps/v1" +"kind": "DaemonSet" +"metadata": + "labels": + "k8s-app": "aws-node" + "name": "aws-node" + "namespace": "kube-system" +"spec": + "selector": + "matchLabels": + "k8s-app": "aws-node" + "template": + "metadata": + "labels": + "k8s-app": "aws-node" + "spec": + "affinity": + "nodeAffinity": + "requiredDuringSchedulingIgnoredDuringExecution": + "nodeSelectorTerms": + - "matchExpressions": + - "key": "kubernetes.io/os" + "operator": "In" + "values": + - "linux" + - "key": "kubernetes.io/arch" + "operator": "In" + "values": + - "amd64" + - "arm64" + - "key": "eks.amazonaws.com/compute-type" + "operator": "NotIn" + "values": + - "fargate" + "containers": + - "env": + - "name": "ADDITIONAL_ENI_TAGS" + "value": "{}" + - "name": "AWS_VPC_CNI_NODE_PORT_SUPPORT" + "value": "true" + - "name": "AWS_VPC_ENI_MTU" + "value": "9001" + - "name": "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_EXTERNALSNAT" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" + "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" + - "name": "AWS_VPC_K8S_CNI_RANDOMIZESNAT" + "value": "prng" + - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" + "value": "eni" + - "name": "AWS_VPC_K8S_PLUGIN_LOG_FILE" + "value": "/var/log/aws-routed-eni/plugin.log" + - "name": "AWS_VPC_K8S_PLUGIN_LOG_LEVEL" + "value": "DEBUG" + - "name": "DISABLE_INTROSPECTION" + "value": "false" + - "name": "DISABLE_METRICS" + "value": "false" + - "name": "ENABLE_POD_ENI" + "value": "false" + - "name": "MY_NODE_NAME" + "valueFrom": + "fieldRef": + "fieldPath": "spec.nodeName" + - "name": "WARM_ENI_TARGET" + "value": "1" + - "name": "CLUSTER_NAME" + "value": "{{ ClusterName }}" {{- range .Networking.AmazonVPC.Env }} - - name: {{ .Name }} - value: "{{ .Value }}" + - "name": "{{ .Name }}" + "value": "{{ .Value }}" {{- end }} - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" - imagePullPolicy: Always - livenessProbe: - exec: - command: - - /app/grpc-health-probe - - -addr=:50051 - initialDelaySeconds: 60 - name: aws-node - ports: - - containerPort: 61678 - name: metrics - readinessProbe: - exec: - command: - - /app/grpc-health-probe - - -addr=:50051 - initialDelaySeconds: 1 - resources: - requests: - cpu: 10m - securityContext: - capabilities: - add: - - NET_ADMIN - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - - mountPath: /host/var/log/aws-routed-eni - name: log-dir - - mountPath: /var/run/aws-node - name: run-dir - - mountPath: /var/run/dockershim.sock - name: dockershim - - mountPath: /run/xtables.lock - name: xtables-lock - hostNetwork: true - initContainers: - - env: - - name: DISABLE_TCP_EARLY_DEMUX - value: "false" - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5 - imagePullPolicy: Always - name: aws-vpc-cni-init - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - priorityClassName: system-node-critical - serviceAccountName: aws-node - terminationGracePeriodSeconds: 10 - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /opt/cni/bin - name: cni-bin-dir - - hostPath: - path: /etc/cni/net.d - name: cni-net-dir - - hostPath: - path: /var/run/dockershim.sock - name: dockershim - - hostPath: - path: /run/xtables.lock - name: xtables-lock - - hostPath: - path: /var/log/aws-routed-eni - type: DirectoryOrCreate - name: log-dir - - hostPath: - path: /var/run/aws-node - type: DirectoryOrCreate - name: run-dir - updateStrategy: - rollingUpdate: - maxUnavailable: "10%" - type: RollingUpdate + "image": "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" + "imagePullPolicy": "Always" + "livenessProbe": + "exec": + "command": + - "/app/grpc-health-probe" + - "-addr=:50051" + "initialDelaySeconds": 60 + "name": "aws-node" + "ports": + - "containerPort": 61678 + "name": "metrics" + "readinessProbe": + "exec": + "command": + - "/app/grpc-health-probe" + - "-addr=:50051" + "initialDelaySeconds": 1 + "resources": + "requests": + "cpu": "10m" + "securityContext": + "capabilities": + "add": + - "NET_ADMIN" + "volumeMounts": + - "mountPath": "/host/opt/cni/bin" + "name": "cni-bin-dir" + - "mountPath": "/host/etc/cni/net.d" + "name": "cni-net-dir" + - "mountPath": "/host/var/log/aws-routed-eni" + "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" + - "mountPath": "/var/run/dockershim.sock" + "name": "dockershim" + - "mountPath": "/run/xtables.lock" + "name": "xtables-lock" + "hostNetwork": true + "initContainers": + - "env": + - "name": "DISABLE_TCP_EARLY_DEMUX" + "value": "false" + "image": "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5" + "imagePullPolicy": "Always" + "name": "aws-vpc-cni-init" + "securityContext": + "privileged": true + "volumeMounts": + - "mountPath": "/host/opt/cni/bin" + "name": "cni-bin-dir" + "priorityClassName": "system-node-critical" + "serviceAccountName": "aws-node" + "terminationGracePeriodSeconds": 10 + "tolerations": + - "operator": "Exists" + "volumes": + - "hostPath": + "path": "/opt/cni/bin" + "name": "cni-bin-dir" + - "hostPath": + "path": "/etc/cni/net.d" + "name": "cni-net-dir" + - "hostPath": + "path": "/var/run/dockershim.sock" + "name": "dockershim" + - "hostPath": + "path": "/run/xtables.lock" + "name": "xtables-lock" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" + "updateStrategy": + "rollingUpdate": + "maxUnavailable": "10%" + "type": "RollingUpdate" --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: aws-node - namespace: kube-system +"apiVersion": "v1" +"kind": "ServiceAccount" +"metadata": + "name": "aws-node" + "namespace": "kube-system" +... From 112221e354904994c8a55ae73fcb6cf5b213af11 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 28 Oct 2020 08:56:03 +0200 Subject: [PATCH 8/8] Amazon VPC CNI - update misc --- upup/models/bindata.go | 446 +++++++++--------- .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 2 +- .../amazonvpc/manifest.yaml | 4 +- ...orking.amazon-vpc-routed-eni-k8s-1.16.yaml | 86 ++-- 4 files changed, 275 insertions(+), 263 deletions(-) diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 6f064e4a59..6c8143a549 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -4359,229 +4359,241 @@ func cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s112YamlTemplate() (*as var _cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s116YamlTemplate = []byte(`# Vendored from https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.7/config/v1.7/aws-k8s-cni.yaml --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aws-node -rules: -- apiGroups: - - crd.k8s.amazonaws.com - resources: +"apiVersion": "rbac.authorization.k8s.io/v1" +"kind": "ClusterRoleBinding" +"metadata": + "name": "aws-node" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "ClusterRole" + "name": "aws-node" +"subjects": +- "kind": "ServiceAccount" + "name": "aws-node" + "namespace": "kube-system" +--- +"apiVersion": "rbac.authorization.k8s.io/v1" +"kind": "ClusterRole" +"metadata": + "name": "aws-node" +"rules": +- "apiGroups": + - "crd.k8s.amazonaws.com" + "resources": + - "eniconfigs" + "verbs": + - "get" + - "list" + - "watch" +- "apiGroups": + - "" + "resources": + - "pods" + - "namespaces" + "verbs": + - "list" + - "watch" + - "get" +- "apiGroups": + - "" + "resources": + - "nodes" + "verbs": + - "list" + - "watch" + - "get" + - "update" +- "apiGroups": + - "extensions" + - "apps" + "resources": - "*" - verbs: - - "*" -- apiGroups: [""] - resources: - - pods - - namespaces - verbs: ["list", "watch", "get"] -- apiGroups: [""] - resources: - - nodes - verbs: ["list", "watch", "get", "update"] -- apiGroups: ["extensions", "apps"] - resources: - - daemonsets - verbs: ["list", "watch"] - + "verbs": + - "list" + - "watch" --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: aws-node - namespace: kube-system - +"apiVersion": "apiextensions.k8s.io/v1beta1" +"kind": "CustomResourceDefinition" +"metadata": + "name": "eniconfigs.crd.k8s.amazonaws.com" +"spec": + "group": "crd.k8s.amazonaws.com" + "names": + "kind": "ENIConfig" + "plural": "eniconfigs" + "singular": "eniconfig" + "scope": "Cluster" + "versions": + - "name": "v1alpha1" + "served": true + "storage": true --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aws-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node -subjects: -- kind: ServiceAccount - name: aws-node - namespace: kube-system - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: aws-node - name: aws-node - namespace: kube-system - labels: - k8s-app: aws-node -spec: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: "10%" - selector: - matchLabels: - k8s-app: aws-node - template: - metadata: - labels: - k8s-app: aws-node - spec: - priorityClassName: system-node-critical - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "kubernetes.io/os" - operator: In - values: - - linux - - key: "kubernetes.io/arch" - operator: In - values: - - amd64 - - arm64 - - key: "eks.amazonaws.com/compute-type" - operator: NotIn - values: - - fargate - serviceAccountName: aws-node - hostNetwork: true - tolerations: - - operator: Exists - containers: - - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" - imagePullPolicy: Always - ports: - - containerPort: 61678 - name: metrics - name: aws-node - readinessProbe: - exec: - command: ["/app/grpc-health-probe", "-addr=:50051"] - initialDelaySeconds: 1 - livenessProbe: - exec: - command: ["/app/grpc-health-probe", "-addr=:50051"] - initialDelaySeconds: 60 - env: - - name: CLUSTER_NAME - value: {{ ClusterName }} - - name: ADDITIONAL_ENI_TAGS - value: '{}' - - name: AWS_VPC_CNI_NODE_PORT_SUPPORT - value: "true" - - name: AWS_VPC_ENI_MTU - value: "9001" - - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER - value: "false" - - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG - value: "false" - - name: AWS_VPC_K8S_CNI_EXTERNALSNAT - value: "false" - - name: AWS_VPC_K8S_CNI_LOGLEVEL - value: DEBUG - - name: AWS_VPC_K8S_CNI_LOG_FILE - value: /host/var/log/aws-routed-eni/ipamd.log - - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT - value: prng - - name: AWS_VPC_K8S_PLUGIN_LOG_FILE - value: /var/log/aws-routed-eni/plugin.log - - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL - value: DEBUG - - name: DISABLE_INTROSPECTION - value: "false" - - name: DISABLE_METRICS - value: "false" - - name: ENABLE_POD_ENI - value: "false" - - name: AWS_VPC_K8S_CNI_VETHPREFIX - value: eni - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: WARM_ENI_TARGET - value: "1" +"apiVersion": "apps/v1" +"kind": "DaemonSet" +"metadata": + "labels": + "k8s-app": "aws-node" + "name": "aws-node" + "namespace": "kube-system" +"spec": + "selector": + "matchLabels": + "k8s-app": "aws-node" + "template": + "metadata": + "labels": + "k8s-app": "aws-node" + "spec": + "affinity": + "nodeAffinity": + "requiredDuringSchedulingIgnoredDuringExecution": + "nodeSelectorTerms": + - "matchExpressions": + - "key": "kubernetes.io/os" + "operator": "In" + "values": + - "linux" + - "key": "kubernetes.io/arch" + "operator": "In" + "values": + - "amd64" + - "arm64" + - "key": "eks.amazonaws.com/compute-type" + "operator": "NotIn" + "values": + - "fargate" + "containers": + - "env": + - "name": "ADDITIONAL_ENI_TAGS" + "value": "{}" + - "name": "AWS_VPC_CNI_NODE_PORT_SUPPORT" + "value": "true" + - "name": "AWS_VPC_ENI_MTU" + "value": "9001" + - "name": "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_EXTERNALSNAT" + "value": "false" + - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" + "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" + - "name": "AWS_VPC_K8S_CNI_RANDOMIZESNAT" + "value": "prng" + - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" + "value": "eni" + - "name": "AWS_VPC_K8S_PLUGIN_LOG_FILE" + "value": "/var/log/aws-routed-eni/plugin.log" + - "name": "AWS_VPC_K8S_PLUGIN_LOG_LEVEL" + "value": "DEBUG" + - "name": "DISABLE_INTROSPECTION" + "value": "false" + - "name": "DISABLE_METRICS" + "value": "false" + - "name": "ENABLE_POD_ENI" + "value": "false" + - "name": "MY_NODE_NAME" + "valueFrom": + "fieldRef": + "fieldPath": "spec.nodeName" + - "name": "WARM_ENI_TARGET" + "value": "1" + - "name": "CLUSTER_NAME" + "value": "{{ ClusterName }}" {{- range .Networking.AmazonVPC.Env }} - - name: {{ .Name }} - value: "{{ .Value }}" + - "name": "{{ .Name }}" + "value": "{{ .Value }}" {{- end }} - resources: - requests: - cpu: 10m - securityContext: - capabilities: - add: - - NET_ADMIN - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - - mountPath: /host/var/log/aws-routed-eni - name: log-dir - - mountPath: /var/run/aws-node - name: run-dir - - mountPath: /var/run/dockershim.sock - name: dockershim - - mountPath: /run/xtables.lock - name: xtables-lock - initContainers: - - env: - - name: DISABLE_TCP_EARLY_DEMUX - value: "false" - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5 - imagePullPolicy: Always - name: aws-vpc-cni-init - resources: {} - securityContext: - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - volumes: - - hostPath: - path: /opt/cni/bin - name: cni-bin-dir - - hostPath: - path: /etc/cni/net.d - name: cni-net-dir - - hostPath: - path: /var/run/dockershim.sock - name: dockershim - - hostPath: - path: /run/xtables.lock - name: xtables-lock - - hostPath: - path: /var/log/aws-routed-eni - type: DirectoryOrCreate - name: log-dir - - hostPath: - path: /var/run/aws-node - type: DirectoryOrCreate - name: run-dir - + "image": "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.7.5" }}" + "imagePullPolicy": "Always" + "livenessProbe": + "exec": + "command": + - "/app/grpc-health-probe" + - "-addr=:50051" + "initialDelaySeconds": 60 + "name": "aws-node" + "ports": + - "containerPort": 61678 + "name": "metrics" + "readinessProbe": + "exec": + "command": + - "/app/grpc-health-probe" + - "-addr=:50051" + "initialDelaySeconds": 1 + "resources": + "requests": + "cpu": "10m" + "securityContext": + "capabilities": + "add": + - "NET_ADMIN" + "volumeMounts": + - "mountPath": "/host/opt/cni/bin" + "name": "cni-bin-dir" + - "mountPath": "/host/etc/cni/net.d" + "name": "cni-net-dir" + - "mountPath": "/host/var/log/aws-routed-eni" + "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" + - "mountPath": "/var/run/dockershim.sock" + "name": "dockershim" + - "mountPath": "/run/xtables.lock" + "name": "xtables-lock" + "hostNetwork": true + "initContainers": + - "env": + - "name": "DISABLE_TCP_EARLY_DEMUX" + "value": "false" + "image": "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5" + "imagePullPolicy": "Always" + "name": "aws-vpc-cni-init" + "securityContext": + "privileged": true + "volumeMounts": + - "mountPath": "/host/opt/cni/bin" + "name": "cni-bin-dir" + "priorityClassName": "system-node-critical" + "serviceAccountName": "aws-node" + "terminationGracePeriodSeconds": 10 + "tolerations": + - "operator": "Exists" + "volumes": + - "hostPath": + "path": "/opt/cni/bin" + "name": "cni-bin-dir" + - "hostPath": + "path": "/etc/cni/net.d" + "name": "cni-net-dir" + - "hostPath": + "path": "/var/run/dockershim.sock" + "name": "dockershim" + - "hostPath": + "path": "/run/xtables.lock" + "name": "xtables-lock" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" + "updateStrategy": + "rollingUpdate": + "maxUnavailable": "10%" + "type": "RollingUpdate" --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: eniconfigs.crd.k8s.amazonaws.com -spec: - scope: Cluster - group: crd.k8s.amazonaws.com - versions: - - name: v1alpha1 - served: true - storage: true - names: - plural: eniconfigs - singular: eniconfig - kind: ENIConfig +"apiVersion": "v1" +"kind": "ServiceAccount" +"metadata": + "name": "aws-node" + "namespace": "kube-system" +... `) func cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s116YamlTemplateBytes() ([]byte, error) { diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index f8661050e6..408feccd3b 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -1024,7 +1024,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann versions := map[string]string{ "k8s-1.10": "1.5.0-kops.2", "k8s-1.12": "1.5.5-kops.1", - "k8s-1.16": "1.6.0-kops.1", + "k8s-1.16": "1.7.5-kops.1", } { diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index a3211b5ee9..e3747de7fe 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -105,8 +105,8 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0' manifest: networking.amazon-vpc-routed-eni/k8s-1.16.yaml - manifestHash: c5ef06e0be88bdb1292b63b08eeea836ad2837bd + manifestHash: fc0f1dd17944bfaca32ccf58163bf7db8099abfc name: networking.amazon-vpc-routed-eni selector: role.kubernetes.io/networking: "1" - version: 1.6.0-kops.1 + version: 1.7.5-kops.1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml index cb30120cd2..67c42995bd 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml @@ -1,3 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aws-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node +subjects: +- kind: ServiceAccount + name: aws-node + namespace: kube-system + +--- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -6,9 +21,11 @@ rules: - apiGroups: - crd.k8s.amazonaws.com resources: - - '*' + - eniconfigs verbs: - - '*' + - get + - list + - watch - apiGroups: - "" resources: @@ -31,33 +48,28 @@ rules: - extensions - apps resources: - - daemonsets + - '*' verbs: - list - watch --- -apiVersion: v1 -kind: ServiceAccount +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: aws-node - namespace: kube-system - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aws-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node -subjects: -- kind: ServiceAccount - name: aws-node - namespace: kube-system + name: eniconfigs.crd.k8s.amazonaws.com +spec: + group: crd.k8s.amazonaws.com + names: + kind: ENIConfig + plural: eniconfigs + singular: eniconfig + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true --- @@ -97,8 +109,6 @@ spec: - fargate containers: - env: - - name: CLUSTER_NAME - value: minimal.example.com - name: ADDITIONAL_ENI_TAGS value: '{}' - name: AWS_VPC_CNI_NODE_PORT_SUPPORT @@ -117,6 +127,8 @@ spec: value: /host/var/log/aws-routed-eni/ipamd.log - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT value: prng + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni - name: AWS_VPC_K8S_PLUGIN_LOG_FILE value: /var/log/aws-routed-eni/plugin.log - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL @@ -127,14 +139,14 @@ spec: value: "false" - name: ENABLE_POD_ENI value: "false" - - name: AWS_VPC_K8S_CNI_VETHPREFIX - value: eni - name: MY_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: WARM_ENI_TARGET value: "1" + - name: CLUSTER_NAME + value: minimal.example.com - name: WARM_IP_TARGET value: "10" - name: AWS_VPC_K8S_CNI_LOGLEVEL @@ -185,16 +197,14 @@ spec: image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.7.5 imagePullPolicy: Always name: aws-vpc-cni-init - resources: {} securityContext: privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir priorityClassName: system-node-critical serviceAccountName: aws-node + terminationGracePeriodSeconds: 10 tolerations: - operator: Exists volumes: @@ -225,18 +235,8 @@ spec: --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition +apiVersion: v1 +kind: ServiceAccount metadata: - name: eniconfigs.crd.k8s.amazonaws.com -spec: - group: crd.k8s.amazonaws.com - names: - kind: ENIConfig - plural: eniconfigs - singular: eniconfig - scope: Cluster - versions: - - name: v1alpha1 - served: true - storage: true + name: aws-node + namespace: kube-system