Merge pull request #6358 from justinsb/calico_with_crds

kops 1.12 configuration for calico: use CRDs

upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template

@@ -1,17 +1,176 @@
-{{- $etcd_scheme := EtcdScheme }}
+---
+# Create all the CustomResourceDefinitions needed for
+# Calico policy and networking mode.
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: felixconfigurations.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: FelixConfiguration
+    plural: felixconfigurations
+    singular: felixconfiguration
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: bgppeers.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPPeer
+    plural: bgppeers
+    singular: bgppeer
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: bgpconfigurations.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPConfiguration
+    plural: bgpconfigurations
+    singular: bgpconfiguration
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ippools.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPPool
+    plural: ippools
+    singular: ippool
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: hostendpoints.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: HostEndpoint
+    plural: hostendpoints
+    singular: hostendpoint
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterinformations.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: ClusterInformation
+    plural: clusterinformations
+    singular: clusterinformation
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworkpolicies.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkPolicy
+    plural: globalnetworkpolicies
+    singular: globalnetworkpolicy
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworksets.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkSet
+    plural: globalnetworksets
+    singular: globalnetworkset
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networkpolicies.crd.projectcalico.org
+  labels:
+    role.kubernetes.io/networking: "1"
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkPolicy
+    plural: networkpolicies
+    singular: networkpolicy
+
+---
+
 # This ConfigMap is used to configure a self-hosted Calico installation.
 kind: ConfigMap
 apiVersion: v1
 metadata:
   name: calico-config
   namespace: kube-system
+  labels:
+    role.kubernetes.io/networking: "1"
 data:
-  # The calico-etcd PetSet service IP:port
+  # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas
-  etcd_endpoints: "{{ $cluster := index .EtcdClusters 0 -}}
+  # below.  We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is
-                      {{- range $j, $member := $cluster.Members -}}
+  # essential.
-                          {{- if $j }},{{ end -}}
+  typha_service_name: "none"
-                          {{ $etcd_scheme }}://etcd-{{ $member.Name }}.internal.{{ ClusterName }}:4001
+
-                      {{- end }}"
+  # TODO: Do we want to configure this?
+  # Configure the MTU to use
+  veth_mtu: "1440"
 
   # Configure the Calico backend to use.
   calico_backend: "bird"
@@ -24,16 +183,13 @@ data:
       "plugins": [
         {
           "type": "calico",
-          "etcd_endpoints": "__ETCD_ENDPOINTS__",
-          {{- if eq $etcd_scheme "https" }}
-          "etcd_ca_cert_file": "/srv/kubernetes/calico/ca.pem",
-          "etcd_cert_file": "/srv/kubernetes/calico/calico-client.pem",
-          "etcd_key_file": "/srv/kubernetes/calico/calico-client-key.pem",
-          "etcd_scheme": "https",
-          {{- end }}
           "log_level": "info",
+          "datastore_type": "kubernetes",
+          "nodename": "__KUBERNETES_NODE_NAME__",
+          "mtu": __CNI_MTU__,
           "ipam": {
-            "type": "calico-ipam"
+            "type": "host-local",
+            "subnet": "usePodCidr"
           },
           "policy": {
             "type": "k8s"
@@ -52,6 +208,17 @@ data:
 
 ---
 
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    role.kubernetes.io/networking: "1"
+---
+
+# Include a clusterrole for the calico-node DaemonSet,
+# and bind it to the calico-node serviceaccount.
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -59,6 +226,7 @@ metadata:
   labels:
     role.kubernetes.io/networking: "1"
 rules:
+  # The CNI plugin needs to get pods, nodes, and namespaces.
   - apiGroups: [""]
     resources:
       - pods
@@ -66,83 +234,137 @@ rules:
       - namespaces
     verbs:
       - get
----
+  - apiGroups: [""]
-
+    resources:
-apiVersion: v1
+      - endpoints
-kind: ServiceAccount
+      - services
-metadata:
+    verbs:
-  name: calico-node
+      # Used to discover service IPs for advertisement.
-  namespace: kube-system
+      - watch
-  labels:
+      - list
-    role.kubernetes.io/networking: "1"
+      # Used to discover Typhas.
----
+      - get
-
+  - apiGroups: [""]
-kind: ClusterRoleBinding
+    resources:
-apiVersion: rbac.authorization.k8s.io/v1
+      - nodes/status
-metadata:
+    verbs:
-  name: calico-node
+      # Needed for clearing NodeNetworkUnavailable flag.
-  labels:
+      - patch
-    role.kubernetes.io/networking: "1"
+      # Calico stores some configuration information in node annotations.
-roleRef:
+      - update
-  apiGroup: rbac.authorization.k8s.io
+  # Watch for changes to Kubernetes NetworkPolicies.
-  kind: ClusterRole
+  - apiGroups: ["networking.k8s.io"]
-  name: calico-node
+    resources:
-subjects:
+      - networkpolicies
-- kind: ServiceAccount
+    verbs:
-  name: calico-node
+      - watch
-  namespace: kube-system
+      - list
----
+  # Used by Calico for policy information.
-
+  - apiGroups: [""]
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    role.kubernetes.io/networking: "1"
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: calico-kube-controllers
-  labels:
-    role.kubernetes.io/networking: "1"
-rules:
-  - apiGroups:
-    - ""
-    - extensions
     resources:
       - pods
       - namespaces
+      - serviceaccounts
+    verbs:
+      - list
+      - watch
+  # The CNI plugin patches pods/status.
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - patch
+  # Calico monitors various CRDs for config.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - globalfelixconfigs
+      - felixconfigurations
+      - bgppeers
+      - globalbgpconfigs
+      - bgpconfigurations
+      - ippools
+      - globalnetworkpolicies
+      - globalnetworksets
       - networkpolicies
+      - clusterinformations
+      - hostendpoints
+    verbs:
+      - get
+      - list
+      - watch
+  # Calico must create and update some CRDs on startup.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ippools
+      - felixconfigurations
+      - clusterinformations
+    verbs:
+      - create
+      - update
+  # Calico stores some configuration information on the node.
+  - apiGroups: [""]
+    resources:
       - nodes
     verbs:
-      - watch
+      - get
       - list
-  - apiGroups:
+      - watch
-    - networking.k8s.io
+  # These permissions are only requried for upgrade from v2.6, and can
+  # be removed after upgrade or on fresh installations.
+  - apiGroups: ["crd.projectcalico.org"]
     resources:
-      - networkpolicies
+      - bgpconfigurations
+      - bgppeers
     verbs:
-      - watch
+      - create
-      - list
+      - update
 ---
-
-kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
 metadata:
-  name: calico-kube-controllers
+  name: calico-node
   labels:
     role.kubernetes.io/networking: "1"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: calico-kube-controllers
+  name: calico-node
 subjects:
 - kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system
+---
+
+# This manifest scales the Calico Kubernetes controllers down to size 0.
+# They are not needed in CRD mode.
+# See https://github.com/projectcalico/kube-controllers
+apiVersion: apps/v1
+kind: Deployment
+metadata:
   name: calico-kube-controllers
   namespace: kube-system
+  labels:
+    k8s-app: calico-kube-controllers
+    role.kubernetes.io/networking: "1"
+spec:
+  # The controllers can only have a single active instance.
+  replicas: 0
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: calico-kube-controllers
+      namespace: kube-system
+      labels:
+        k8s-app: calico-kube-controllers
+        role.kubernetes.io/networking: "1"
+    spec:
+      containers:
+        - name: calico-kube-controllers
+          image: quay.io/calico/kube-controllers:v3.4.0
+      initContainers:
+        - name: migrate
+          image: calico/upgrade:v1.0.5
 
 ---
 
@@ -187,6 +409,41 @@ spec:
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0
+      initContainers:
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: quay.io/calico/cni:v3.4.0
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            # Set the hostname based on the k8s node name.
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # CNI MTU Config variable
+            - name: CNI_MTU
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: veth_mtu
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
       containers:
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
@@ -194,20 +451,23 @@ spec:
         - name: calico-node
           image: quay.io/calico/node:v3.4.0
           env:
-            # The location of the Calico etcd cluster.
+            # Use Kubernetes API as the backing datastore.
-            - name: ETCD_ENDPOINTS
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+            # Typha support: controlled by the ConfigMap.
+            - name: FELIX_TYPHAK8SSERVICENAME
               valueFrom:
                 configMapKeyRef:
                   name: calico-config
-                  key: etcd_endpoints
+                  key: typha_service_name
-            {{- if eq $etcd_scheme "https" }}
+            # Wait for the datastore.
-            - name: ETCD_CERT_FILE
+            - name: WAIT_FOR_DATASTORE
-              value: /certs/calico-client.pem
+              value: "true"
-            - name: ETCD_KEY_FILE
+            # Set based on the k8s node name.
-              value: /certs/calico-client-key.pem
+            - name: NODENAME
-            - name: ETCD_CA_CERT_FILE
+              valueFrom:
-              value: /certs/ca.pem
+                fieldRef:
-            {{- end }}
+                  fieldPath: spec.nodeName
             # Choose the backend to use.
             - name: CALICO_NETWORKING_BACKEND
               valueFrom:
@@ -216,32 +476,42 @@ spec:
                   key: calico_backend
             # Cluster type to identify the deployment type
             - name: CLUSTER_TYPE
+              # was value: "k8s,bgp"
               value: "kops,bgp"
-            # Disable file logging so `kubectl logs` works.
+            # Auto-detect the BGP IP address.
-            - name: CALICO_DISABLE_FILE_LOGGING
+            - name: IP
-              value: "true"
+              value: "autodetect"
-            # Set noderef for node controller.
+            # Enable IPIP
-            - name: CALICO_K8S_NODE_REF
+            - name: CALICO_IPV4POOL_IPIP
+              # was value: "Always"
+              value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}cross-subnet{{- else -}}always{{- end -}}"
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
               valueFrom:
-                fieldRef:
+                configMapKeyRef:
-                  fieldPath: spec.nodeName
+                  name: calico-config
-            # Set Felix endpoint to host default action to ACCEPT.
+                  key: veth_mtu
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
             # The default IPv4 pool to create on startup if none exists. Pod IPs will be
             # chosen from this range. Changing this value after installation will have
             # no effect. This should fall within `--cluster-cidr`.
-            # Configure the IP Pool from which Pod IPs will be chosen.
             - name: CALICO_IPV4POOL_CIDR
               value: "{{ .KubeControllerManager.ClusterCIDR }}"
-            - name: CALICO_IPV4POOL_IPIP
+            # Disable file logging so `kubectl logs` works.
-              value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}cross-subnet{{- else -}}always{{- end -}}"
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
             # Disable IPv6 on Kubernetes.
             - name: FELIX_IPV6SUPPORT
               value: "false"
             # Set Felix logging to the desired level
             - name: FELIX_LOGSEVERITYSCREEN
               value: "{{- or .Networking.Calico.LogSeverityScreen "info" }}"
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+
+            # kops additions
             # Set to enable the experimental Prometheus metrics server
             - name: FELIX_PROMETHEUSMETRICSENABLED
               value: "{{- or .Networking.Calico.PrometheusMetricsEnabled "false" }}"
@@ -254,11 +524,13 @@ spec:
             # Enable Prometheus process metrics collection
             - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED
               value: "{{- or .Networking.Calico.PrometheusProcessMetricsEnabled "true" }}"
-            # Auto-detect the BGP IP address.
+
-            - name: IP
+            # TODO: In previous versions, no longer needed?
-              value: "autodetect"
+            # Set noderef for node controller.
-            - name: FELIX_HEALTHENABLED
+            #- name: CALICO_K8S_NODE_REF
-              value: "true"
+            #  valueFrom:
+            #    fieldRef:
+            #      fieldPath: spec.nodeName
           securityContext:
             privileged: true
           resources:
@@ -283,95 +555,20 @@ spec:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
+            - mountPath: /run/xtables.lock
+              name: xtables-lock
+              readOnly: false
             - mountPath: /var/run/calico
               name: var-run-calico
               readOnly: false
             - mountPath: /var/lib/calico
               name: var-lib-calico
               readOnly: false
+            # TODO: Still needed?
             # Necessary for gossip based DNS
             - mountPath: /etc/hosts
               name: etc-hosts
               readOnly: true
-            {{- if eq $etcd_scheme "https" }}
-            - mountPath: /certs
-              name: calico
-              readOnly: true
-            {{- end }}
-        # This container installs the Calico CNI binaries
-        # and CNI network config file on each node.
-        - name: install-cni
-          image: quay.io/calico/cni:v3.4.0
-          command: ["/install-cni.sh"]
-          env:
-            # Name of the CNI config file to create.
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            # The location of the Calico etcd cluster.
-            - name: ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-            # The CNI network config to install on each node.
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-            # Necessary for gossip based DNS
-            - mountPath: /etc/hosts
-              name: etc-hosts
-              readOnly: true
-          resources:
-            requests:
-              cpu: 10m
-      initContainers:
-        - name: migrate
-          image: calico/upgrade:v1.0.5
-          command: ['/bin/sh', '-c', '/node-init-container.sh']
-          env:
-            - name: CALICO_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-            - name: CALICO_APIV1_DATASTORE_TYPE
-              value: "etcdv2"
-            - name: CALICO_APIV1_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-          {{- if eq $etcd_scheme "https" }}
-            - name: CALICO_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-            - name: CALICO_APIV1_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_APIV1_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_APIV1_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-          {{- end }}
-          volumeMounts:
-            # Necessary for gossip based DNS
-            - mountPath: /etc/hosts
-              name: etc-hosts
-              readOnly: true
-          {{- if eq $etcd_scheme "https" }}
-            - mountPath: /certs
-              name: calico
-              readOnly: true
-          {{- end }}
       volumes:
         # Used by calico/node.
         - name: lib-modules
@@ -383,6 +580,10 @@ spec:
         - name: var-lib-calico
           hostPath:
             path: /var/lib/calico
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
         # Used to install CNI.
         - name: cni-bin-dir
           hostPath:
@@ -390,265 +591,14 @@ spec:
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d
+        # TODO: Still needed?
         # Necessary for gossip based DNS
         - name: etc-hosts
           hostPath:
             path: /etc/hosts
-        {{- if eq $etcd_scheme "https" }}
-        - name: calico
-          hostPath:
-            path: /srv/kubernetes/calico
-        {{- end }}
 
 ---
 
-# This manifest deploys the Calico Kubernetes controllers.
-# See https://github.com/projectcalico/kube-controllers
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    k8s-app: calico-kube-controllers
-    role.kubernetes.io/networking: "1"
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
-spec:
-  # The controllers can only have a single active instance.
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      k8s-app: calico-kube-controllers
-      role.kubernetes.io/networking: "1"
-  template:
-    metadata:
-      name: calico-kube-controllers
-      namespace: kube-system
-      labels:
-        k8s-app: calico-kube-controllers
-        role.kubernetes.io/networking: "1"
-    spec:
-      # The controllers must run in the host network namespace so that
-      # it isn't governed by policy that would prevent it from working.
-      hostNetwork: true
-      tolerations:
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      serviceAccountName: calico-kube-controllers
-      containers:
-        - name: calico-kube-controllers
-          image: quay.io/calico/kube-controllers:v3.4.0
-          resources:
-            requests:
-              cpu: 10m
-          env:
-            # The location of the Calico etcd cluster.
-            - name: ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-            # Choose which controllers to run.
-            - name: ENABLED_CONTROLLERS
-              value: policy,profile,workloadendpoint,node
-            {{- if eq $etcd_scheme "https" }}
-            - name: ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-          volumeMounts:
-            - mountPath: /certs
-              name: calico
-              readOnly: true
-            {{- end }}
-          readinessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -r
-      initContainers:
-        - name: migrate
-          image: calico/upgrade:v1.0.5
-          command: ['/bin/sh', '-c', '/controller-init.sh']
-          env:
-            - name: CALICO_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-            - name: CALICO_APIV1_DATASTORE_TYPE
-              value: "etcdv2"
-            - name: CALICO_APIV1_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-          {{- if eq $etcd_scheme "https" }}
-            - name: CALICO_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-            - name: CALICO_APIV1_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_APIV1_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_APIV1_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-          {{- end }}
-          volumeMounts:
-            # Necessary for gossip based DNS
-            - mountPath: /etc/hosts
-              name: etc-hosts
-              readOnly: true
-          {{- if eq $etcd_scheme "https" }}
-            - mountPath: /certs
-              name: calico
-              readOnly: true
-          {{- end }}
-      volumes:
-        # Necessary for gossip based DNS
-        - name: etc-hosts
-          hostPath:
-            path: /etc/hosts
-        {{- if eq $etcd_scheme "https" }}
-        - name: calico
-          hostPath:
-            path: /srv/kubernetes/calico
-        {{- end }}
-
-# This manifest runs the Migration complete container that monitors for the
-# completion of the calico-node Daemonset rollout and when it finishes
-# successfully rolling out it will mark the migration complete and allow pods
-# to be created again.
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-upgrade-job
-  namespace: kube-system
-  labels:
-    role.kubernetes.io/networking: "1"
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: calico-upgrade-job
-  labels:
-    role.kubernetes.io/networking: "1"
-rules:
-  - apiGroups:
-      - extensions
-    resources:
-      - daemonsets
-      - daemonsets/status
-    verbs:
-      - get
-      - list
-      - watch
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-upgrade-job
-  labels:
-    role.kubernetes.io/networking: "1"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-upgrade-job
-subjects:
-- kind: ServiceAccount
-  name: calico-upgrade-job
-  namespace: kube-system
----
-# If anything in this job is changed then the name of the job
-# should be changed because Jobs cannot be updated, so changing
-# the name would run a different Job if the previous version had been
-# created before and it does not hurt to rerun this job.
-
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: calico-complete-upgrade-v331
-  namespace: kube-system
-  labels:
-    role.kubernetes.io/networking: "1"
-spec:
-  template:
-    metadata:
-      labels:
-        role.kubernetes.io/networking: "1"
-    spec:
-      hostNetwork: true
-      serviceAccountName: calico-upgrade-job
-      restartPolicy: OnFailure
-      containers:
-        - name: migrate-completion
-          image: calico/upgrade:v1.0.5
-          command: ['/bin/sh', '-c', '/completion-job.sh']
-          env:
-            - name: EXPECTED_NODE_IMAGE
-              value: quay.io/calico/node:v3.4.0
-            # The location of the Calico etcd cluster.
-            - name: CALICO_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-            - name: CALICO_APIV1_DATASTORE_TYPE
-              value: "etcdv2"
-            - name: CALICO_APIV1_ETCD_ENDPOINTS
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: etcd_endpoints
-          {{- if eq $etcd_scheme "https" }}
-            - name: CALICO_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-            - name: CALICO_APIV1_ETCD_CERT_FILE
-              value: /certs/calico-client.pem
-            - name: CALICO_APIV1_ETCD_KEY_FILE
-              value: /certs/calico-client-key.pem
-            - name: CALICO_APIV1_ETCD_CA_CERT_FILE
-              value: /certs/ca.pem
-          {{- end }}
-          volumeMounts:
-            # Necessary for gossip based DNS
-            - mountPath: /etc/hosts
-              name: etc-hosts
-              readOnly: true
-          {{- if eq $etcd_scheme "https" }}
-            - mountPath: /certs
-              name: calico
-              readOnly: true
-          {{- end }}
-      volumes:
-        - name: etc-hosts
-          hostPath:
-            path: /etc/hosts
-        {{- if eq $etcd_scheme "https" }}
-        - name: calico
-          hostPath:
-            path: /srv/kubernetes/calico
-        {{- end }}
-
 {{ if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}
 # This manifest installs the k8s-ec2-srcdst container, which disables
 # src/dst ip checks to allow BGP to function for calico for hosts within subnets
@@ -751,3 +701,4 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
 {{- end -}}
+

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -810,7 +810,22 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri
 			"k8s-1.6":     "2.6.9-kops.1",
 			"k8s-1.7":     "2.6.12-kops.1",
 			"k8s-1.7-v3":  "3.4.0-kops.3",
-			"k8s-1.12":    "3.4.0-kops.3",
+			"k8s-1.12":    "3.4.0-kops.4",
+		}
+
+		{
+			id := "k8s-1.12"
+			location := key + "/" + id + ".yaml"
+
+			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				Name:              fi.String(key),
+				Version:           fi.String(versions[id]),
+				Selector:          networkingSelector,
+				Manifest:          fi.String(location),
+				KubernetesVersion: ">=1.12.0",
+				Id:                id,
+			})
+			manifests[key+"-"+id] = "addons/" + location
 		}
 
 		if b.cluster.Spec.Networking.Calico.MajorVersion == "v3" {
@@ -828,21 +843,6 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri
 				})
 				manifests[key+"-"+id] = "addons/" + location
 			}
-
-			{
-				id := "k8s-1.12"
-				location := key + "/" + id + ".yaml"
-
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
-					Name:              fi.String(key),
-					Version:           fi.String(versions[id]),
-					Selector:          networkingSelector,
-					Manifest:          fi.String(location),
-					KubernetesVersion: ">=1.12.0",
-					Id:                id,
-				})
-				manifests[key+"-"+id] = "addons/" + location
-			}
 		} else {
 			{
 				id := "pre-k8s-1.6"