Add configuration options for Cilium 1.6

Signed-off-by: Maciej Kwiek <maciej@isovalent.com>

k8s/crds/kops_v1alpha2_cluster.yaml

@@ -2121,6 +2121,8 @@ spec:
                   type: object
                 cilium:
                   properties:
+                    IPTablesRulesNoinstall:
+                      type: boolean
                     accessLog:
                       type: string
                     agentLabels:
@@ -2129,16 +2131,28 @@ spec:
                       type: array
                     allowLocalhost:
                       type: string
+                    autoDirectNodeRoutes:
+                      type: boolean
                     autoIpv6NodeRoutes:
                       type: boolean
+                    bpfCTGlobalAnyMax:
+                      format: int64
+                      type: integer
+                    bpfCTGlobalTCPMax:
+                      format: int64
+                      type: integer
                     bpfRoot:
                       type: string
+                    clusterName:
+                      type: string
                     containerRuntime:
                       items:
                         type: string
                       type: array
                     containerRuntimeEndpoint:
                       type: object
+                    containerRuntimeLabels:
+                      type: string
                     debug:
                       type: boolean
                     debugVerbose:
@@ -2155,10 +2169,16 @@ spec:
                       type: boolean
                     disableMasquerade:
                       type: boolean
+                    enableNodePort:
+                      type: boolean
                     enablePolicy:
                       type: string
                     enableTracing:
                       type: boolean
+                    enableipv4:
+                      type: boolean
+                    enableipv6:
+                      type: boolean
                     envoyLog:
                       type: string
                     ipv4ClusterCidrMaskSize:
@@ -2209,22 +2229,30 @@ spec:
                     logstashProbeTimer:
                       format: int32
                       type: integer
+                    monitorAggregation:
+                      type: string
                     nat46Range:
                       type: string
                     pprof:
                       type: boolean
+                    preallocateBPFMaps:
+                      type: boolean
                     prefilterDevice:
                       type: string
                     prometheusServeAddr:
                       type: string
                     restore:
                       type: boolean
+                    sidecarIstioProxyImage:
+                      type: string
                     singleClusterRoute:
                       type: boolean
                     socketPath:
                       type: string
                     stateDir:
                       type: string
+                    toFqdnsEnablePoller:
+                      type: boolean
                     tracePayloadlen:
                       format: int64
                       type: integer
@@ -2232,6 +2260,22 @@ spec:
                       type: string
                     version:
                       type: string
+                    waitBPFMount:
+                      type: boolean
+                  required:
+                  - enableipv6
+                  - enableipv4
+                  - monitorAggregation
+                  - bpfCTGlobalTCPMax
+                  - bpfCTGlobalAnyMax
+                  - preallocateBPFMaps
+                  - sidecarIstioProxyImage
+                  - clusterName
+                  - toFqdnsEnablePoller
+                  - waitBPFMount
+                  - IPTablesRulesNoinstall
+                  - autoDirectNodeRoutes
+                  - enableNodePort
                   type: object
                 classic:
                   type: object
@@ -2336,6 +2380,9 @@ spec:
                 NonMasqueradeCIDR is the CIDR for the internal k8s network (on which
                 pods & services live) It cannot overlap ServiceClusterIPRange
               type: string
+            podCIDR:
+              description: PodCIDR is the CIDR from which we allocate IPs for pods
+              type: string
             project:
               description: Project is the cloud project we should use, required on
                 GCE

k8s/crds/kops_v1alpha2_instancegroup.yaml

@@ -196,6 +196,10 @@ spec:
             image:
               description: Image is the instance (ami etc) we should use
               type: string
+            instanceProtection:
+              description: InstanceProtection makes new instances in an autoscaling
+                group protected from scale in
+              type: boolean
             kubelet:
               description: Kubelet overrides kubelet config from the ClusterSpec
               properties:
@@ -520,7 +524,8 @@ spec:
                   type: string
                 volumePluginDirectory:
                   description: The full path of the directory in which to search for
-                    additional third party volume plugins
+                    additional third party volume plugins (this path must be writeable,
+                    dependant on your choice of OS)
                   type: string
                 volumeStatsAggPeriod:
                   description: VolumeStatsAggPeriod is the interval for kubelet to

pkg/apis/kops/networking.go

@@ -155,7 +155,7 @@ type AmazonVPCNetworkingSpec struct {
 	ImageName string `json:"imageName,omitempty"`
 }
 
-const CiliumDefaultVersion = "v1.6.0"
+const CiliumDefaultVersion = "v1.6.1"
 
 // CiliumNetworkingSpec declares that we want Cilium networking
 type CiliumNetworkingSpec struct {
@@ -209,6 +209,27 @@ type CiliumNetworkingSpec struct {
 	StateDir                 string            `json:"stateDir,omitempty"`
 	TracePayloadLen          int               `json:"tracePayloadlen,omitempty"`
 	Tunnel                   string            `json:"tunnel,omitempty"`
+
+	EnableIpv6             bool   `json:"enableipv6"`
+	EnableIpv4             bool   `json:"enableipv4"`
+	MonitorAggregation     string `json:"monitorAggregation"`
+	BPFCTGlobalTCPMax      int    `json:"bpfCTGlobalTCPMax"`
+	BPFCTGlobalAnyMax      int    `json:"bpfCTGlobalAnyMax"`
+	PreallocateBPFMaps     bool   `json:"preallocateBPFMaps"`
+	SidecarIstioProxyImage string `json:"sidecarIstioProxyImage"`
+	ClusterName            string `json:"clusterName"`
+	ToFqdnsEnablePoller    bool   `json:"toFqdnsEnablePoller"`
+	ContainerRuntimeLabels string `json:"containerRuntimeLabels,omitempty"`
+	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
+	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
+	EnableNodePort         bool   `json:"enableNodePort"`
+
+	//node init options
+	RemoveCbrBridge       bool   `json:"removeCbrBridge"`
+	RestartPods           bool   `json:"restartPods"`
+	ReconfigureKubelet    bool   `json:"reconfigureKubelet"`
+	NodeInitBootstrapFile string `json:"nodeInitBootstrapFile"`
+	CniBinPath            string `json:"cniBinPath"`
 }
 
 // LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking

pkg/apis/kops/v1alpha1/networking.go

@@ -206,6 +206,27 @@ type CiliumNetworkingSpec struct {
 	StateDir                 string            `json:"stateDir,omitempty"`
 	TracePayloadLen          int               `json:"tracePayloadlen,omitempty"`
 	Tunnel                   string            `json:"tunnel,omitempty"`
+
+	EnableIpv6             bool   `json:"enableipv6"`
+	EnableIpv4             bool   `json:"enableipv4"`
+	MonitorAggregation     string `json:"monitorAggregation"`
+	BPFCTGlobalTCPMax      int    `json:"bpfCTGlobalTCPMax"`
+	BPFCTGlobalAnyMax      int    `json:"bpfCTGlobalAnyMax"`
+	PreallocateBPFMaps     bool   `json:"preallocateBPFMaps"`
+	SidecarIstioProxyImage string `json:"sidecarIstioProxyImage"`
+	ClusterName            string `json:"clusterName"`
+	ToFqdnsEnablePoller    bool   `json:"toFqdnsEnablePoller"`
+	ContainerRuntimeLabels string `json:"containerRuntimeLabels,omitempty"`
+	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
+	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
+	EnableNodePort         bool   `json:"enableNodePort"`
+
+	//node init options
+	RemoveCbrBridge       bool   `json:"removeCbrBridge"`
+	RestartPods           bool   `json:"restartPods"`
+	ReconfigureKubelet    bool   `json:"reconfigureKubelet"`
+	NodeInitBootstrapFile string `json:"nodeInitBootstrapFile"`
+	CniBinPath            string `json:"cniBinPath"`
 }
 
 // LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking

pkg/apis/kops/v1alpha2/networking.go

@@ -207,6 +207,27 @@ type CiliumNetworkingSpec struct {
 	StateDir                 string            `json:"stateDir,omitempty"`
 	TracePayloadLen          int               `json:"tracePayloadlen,omitempty"`
 	Tunnel                   string            `json:"tunnel,omitempty"`
+
+	EnableIpv6             bool   `json:"enableipv6"`
+	EnableIpv4             bool   `json:"enableipv4"`
+	MonitorAggregation     string `json:"monitorAggregation"`
+	BPFCTGlobalTCPMax      int    `json:"bpfCTGlobalTCPMax"`
+	BPFCTGlobalAnyMax      int    `json:"bpfCTGlobalAnyMax"`
+	PreallocateBPFMaps     bool   `json:"preallocateBPFMaps"`
+	SidecarIstioProxyImage string `json:"sidecarIstioProxyImage"`
+	ClusterName            string `json:"clusterName"`
+	ToFqdnsEnablePoller    bool   `json:"toFqdnsEnablePoller"`
+	ContainerRuntimeLabels string `json:"containerRuntimeLabels,omitempty"`
+	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
+	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
+	EnableNodePort         bool   `json:"enableNodePort"`
+
+	//node init options
+	RemoveCbrBridge       bool   `json:"removeCbrBridge"`
+	RestartPods           bool   `json:"restartPods"`
+	ReconfigureKubelet    bool   `json:"reconfigureKubelet"`
+	NodeInitBootstrapFile string `json:"nodeInitBootstrapFile"`
+	CniBinPath            string `json:"cniBinPath"`
 }
 
 // LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template

@@ -4,6 +4,7 @@ metadata:
   name: cilium-config
   namespace: kube-system
 data:
+{{ with .Networking.Cilium }}
   # Identity allocation mode selects how identities are shared between cilium
   # nodes by setting how they are stored. The options are "crd" or "kvstore".
   # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
@@ -16,17 +17,17 @@ data:
   #   setting it to "kvstore".
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
-  debug: "false"
+  debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
-  enable-ipv4: "true"
+  enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
   # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
   # address.
-  enable-ipv6: "false"
+  enable-ipv6: "{{- if .EnableIpv6 -}}true{{- else -}}false{{- end -}}"
   # If you want cilium monitor to aggregate tracing for packets, set this level
   # to "low", "medium", or "maximum". The higher the level, the less packets
   # that will be seen in monitor output.
-  monitor-aggregation: medium
+  monitor-aggregation: "{{- if eq .MonitorAggregation "" -}}medium{{- else -}}{{ .MonitorAggregation }}{{- end -}}"
   # ct-global-max-entries-* specifies the maximum number of connections
   # supported across all endpoints, split by protocol: tcp or other. One pair
   # of maps uses these values for IPv4 connections, and another pair of maps
@@ -38,8 +39,9 @@ data:
   #
   # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
   # during the upgrade process, comment out these options.
-  bpf-ct-global-tcp-max: "524288"
+  bpf-ct-global-tcp-max: "{{- if eq .BPFCTGlobalTCPMax 0 -}}524288{{- else -}}{{ .BPFCTGlobalTCPMax}}{{- end -}}"
-  bpf-ct-global-any-max: "262144"
+  bpf-ct-global-any-max: "{{- if eq .BPFCTGlobalAnyMax 0 -}}262144{{- else -}}{{ .BPFCTGlobalAnyMax}}{{- end -}}"
+
   # Pre-allocation of map entries allows per-packet latency to be reduced, at
   # the expense of up-front memory allocation for the entries in the maps. The
   # default value below will minimize memory usage in the default installation;
@@ -56,28 +58,20 @@ data:
   #
   # If this option is set to "false" during an upgrade from 1.3 or earlier to
   # 1.4 or later, then it may cause one-time disruptions during the upgrade.
-  preallocate-bpf-maps: "false"
+  preallocate-bpf-maps: "{{- if .PreallocateBPFMaps -}}true{{- else -}}false{{- end -}}"
   # Regular expression matching compatible Istio sidecar istio-proxy
   # container image names
-  sidecar-istio-proxy-image: "cilium/istio_proxy"
+  sidecar-istio-proxy-image: "{{- if eq .SidecarIstioProxyImage "" -}}cilium/istio_proxy{{- else -}}{{ .SidecarIstioProxyImage }}{{- end -}}"
   # Encapsulation mode for communication between nodes
   # Possible values:
   #   - disabled
   #   - vxlan (default)
   #   - geneve
-  tunnel: vxlan
+  tunnel: "{{- if eq .Tunnel "" -}}vxlan{{- else -}}{{ .Tunnel }}{{- end -}}"
+
   # Name of the cluster. Only relevant when building a mesh of clusters.
-  cluster-name: default
+  cluster-name: "{{- if eq .ClusterName "" -}}default{{- else -}}{{ .ClusterName}}{{- end -}}"
-  # Unique ID of the cluster. Must be unique across all conneted clusters and
+
-  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
-  # DNS Polling periodically issues a DNS lookup for each `matchName` from
-  # cilium-agent. The result is used to regenerate endpoint policy.
-  # DNS lookups are repeated with an interval of 5 seconds, and are made for
-  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
-  # data is used instead. An IP change will trigger a regeneration of the Cilium
-  # policy for each endpoint and increment the per cilium-agent policy
-  # repository revision.
-  #
   # This option is disabled by default starting from version 1.4.x in favor
   # of a more powerful DNS proxy-based implementation, see [0] for details.
   # Enable this option if you want to use FQDN policies but do not want to use
@@ -89,9 +83,9 @@ data:
   #
   # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
   # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
-  tofqdns-enable-poller: "false"
+  tofqdns-enable-poller: "{{- if .ToFqdnsEnablePoller -}}true{{- else -}}false{{- end -}}"
   # wait-bpf-mount makes init container wait until bpf filesystem is mounted
-  wait-bpf-mount: "false"
+  wait-bpf-mount: "true"
   # Enable fetching of container-runtime specific metadata
   #
   # By default, the Kubernetes pod and namespace labels are retrieved and
@@ -111,11 +105,12 @@ data:
   # - none
   # - auto (automatically detect the container runtime)
   #
-  container-runtime: none
+  container-runtime: "{{- if eq .ContainerRuntimeLabels "" -}}none{{- else -}}{{ .ContainerRuntimeLabels }}{{- end -}}"
-  masquerade: "true"
+  masquerade: "{{- if .DisableMasquerade -}}false{{- else -}}true{{- end -}}"
-  install-iptables-rules: "true"
+  install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
-  auto-direct-node-routes: "false"
+  auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
-  enable-node-port: "false"
+  enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
+{{ end }} # With .Networking.Cilium end
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -622,3 +617,110 @@ spec:
       restartPolicy: Always
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: cilium-node-init
+  namespace: kube-system
+  labels:
+    app: cilium-node-init
+spec:
+  selector:
+    matchLabels:
+      app: cilium-node-init
+  template:
+    metadata:
+      labels:
+        app: cilium-node-init
+    spec:
+      tolerations:
+      - operator: Exists
+      hostPID: true
+      hostNetwork: true
+      containers:
+        - name: node-init
+          image: gcr.io/google-containers/startup-script:v1
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: true
+          env:
+          # STARTUP_SCRIPT is the script run on node bootstrap. Node
+          # bootstrapping can be customized in this script.
+          - name: STARTUP_SCRIPT
+            value: |
+              #!/bin/bash
+              set -o errexit
+              set -o pipefail
+              set -o nounset
+              mount | grep "/sys/fs/bpf type bpf" || {
+                # Mount the filesystem until next reboot
+                echo "Mounting BPF filesystem..."
+                mount bpffs /sys/fs/bpf -t bpf
+              }
+              echo "Installing BPF filesystem mount"
+              cat >/tmp/sys-fs-bpf.mount <<EOF
+              [Unit]
+              Description=Mount BPF filesystem (Cilium)
+              Documentation=http://docs.cilium.io/
+              DefaultDependencies=no
+              Before=local-fs.target umount.target
+              After=swap.target
+              [Mount]
+              What=bpffs
+              Where=/sys/fs/bpf
+              Type=bpf
+              [Install]
+              WantedBy=multi-user.target
+              EOF
+              if [ -d "/etc/systemd/system/" ]; then
+                mv /tmp/sys-fs-bpf.mount /etc/systemd/system/
+                echo "Installed sys-fs-bpf.mount to /etc/systemd/system/"
+              elif [ -d "/lib/systemd/system/" ]; then
+                mv /tmp/sys-fs-bpf.mount /lib/systemd/system/
+                echo "Installed sys-fs-bpf.mount to /lib/systemd/system/"
+              fi
+              # Azure specific:
+              if [ -f /var/run/azure-vnet.json ]; then
+                sed -i 's/"Mode": "bridge",/"Mode": "transparent",/g' /var/run/azure-vnet.json
+              fi
+              # Ensure that filesystem gets mounted on next reboot
+              systemctl enable sys-fs-bpf.mount
+              systemctl start sys-fs-bpf.mount
+              echo "Link information:"
+              ip link
+              echo "Routing table:"
+              ip route
+              echo "Addressing:"
+              ip -4 a
+              ip -6 a
+              {{ with .Networking.Cilium }}
+              {{ if .RemoveCbrBridge }}
+              if ip link show cbr0; then
+                echo "Detected cbr0 bridge. Deleting interface..."
+                ip link del cbr0
+              fi
+              {{ end }}
+              {{ if .RestartPods }}
+              echo "Restarting kubenet managed pods"
+              if grep -q 'docker' /etc/crictl.yaml; then
+                # Works for COS, ubuntu
+                for f in `find /var/lib/cni/networks/ -type f ! -name lock ! -name last_reserved_ip.0`; do docker rm -f $(cat $f) || true; done
+              else
+                # COS-beta (with containerd)
+                for f in `find /var/lib/cni/networks/ -type f ! -name lock ! -name last_reserved_ip.0`; do crictl stopp $(cat $f) || true; done
+              fi
+              {{ end }}
+              {{ if .ReconfigureKubelet }}
+              # GKE: Alter the kubelet configuration to run in CNI mode
+              echo "Changing kubelet configuration to --network-plugin=cni --cni-bin-dir={{ .CniBinPath }}"
+              mkdir -p {{ .CniBinPath }}
+              sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir={{ .CniBinPath }}:g" /etc/default/kubelet
+              echo "Restarting kubelet..."
+              systemctl restart kubelet
+              {{ end }}
+              {{ if not (eq .NodeInitBootstrapFile "") }}
+              date > {{ .Values.global.nodeinit.bootstrapFile }}
+              {{ end }}
+              {{ end }}#end of with .Networking.Cilium
+              echo "Node initialization complete"

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template

@@ -4,6 +4,7 @@ metadata:
   name: cilium-config
   namespace: kube-system
 data:
+{{ with .Networking.Cilium }}
   # Identity allocation mode selects how identities are shared between cilium
   # nodes by setting how they are stored. The options are "crd" or "kvstore".
   # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
@@ -16,17 +17,17 @@ data:
   #   setting it to "kvstore".
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
-  debug: "false"
+  debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
-  enable-ipv4: "true"
+  enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
   # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
   # address.
-  enable-ipv6: "false"
+  enable-ipv6: "{{- if .EnableIpv6 -}}true{{- else -}}false{{- end -}}"
   # If you want cilium monitor to aggregate tracing for packets, set this level
   # to "low", "medium", or "maximum". The higher the level, the less packets
   # that will be seen in monitor output.
-  monitor-aggregation: medium
+  monitor-aggregation: "{{- if eq .MonitorAggregation "" -}}medium{{- else -}}{{ .MonitorAggregation }}{{- end -}}"
   # ct-global-max-entries-* specifies the maximum number of connections
   # supported across all endpoints, split by protocol: tcp or other. One pair
   # of maps uses these values for IPv4 connections, and another pair of maps
@@ -38,8 +39,9 @@ data:
   #
   # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
   # during the upgrade process, comment out these options.
-  bpf-ct-global-tcp-max: "524288"
+  bpf-ct-global-tcp-max: "{{- if eq .BPFCTGlobalTCPMax 0 -}}524288{{- else -}}{{ .BPFCTGlobalTCPMax}}{{- end -}}"
-  bpf-ct-global-any-max: "262144"
+  bpf-ct-global-any-max: "{{- if eq .BPFCTGlobalAnyMax 0 -}}262144{{- else -}}{{ .BPFCTGlobalAnyMax}}{{- end -}}"
+
   # Pre-allocation of map entries allows per-packet latency to be reduced, at
   # the expense of up-front memory allocation for the entries in the maps. The
   # default value below will minimize memory usage in the default installation;
@@ -56,28 +58,20 @@ data:
   #
   # If this option is set to "false" during an upgrade from 1.3 or earlier to
   # 1.4 or later, then it may cause one-time disruptions during the upgrade.
-  preallocate-bpf-maps: "false"
+  preallocate-bpf-maps: "{{- if .PreallocateBPFMaps -}}true{{- else -}}false{{- end -}}"
   # Regular expression matching compatible Istio sidecar istio-proxy
   # container image names
-  sidecar-istio-proxy-image: "cilium/istio_proxy"
+  sidecar-istio-proxy-image: "{{- if eq .SidecarIstioProxyImage "" -}}cilium/istio_proxy{{- else -}}{{ .SidecarIstioProxyImage }}{{- end -}}"
   # Encapsulation mode for communication between nodes
   # Possible values:
   #   - disabled
   #   - vxlan (default)
   #   - geneve
-  tunnel: vxlan
+  tunnel: "{{- if eq .Tunnel "" -}}vxlan{{- else -}}{{ .Tunnel }}{{- end -}}"
+
   # Name of the cluster. Only relevant when building a mesh of clusters.
-  cluster-name: default
+  cluster-name: "{{- if eq .ClusterName "" -}}default{{- else -}}{{ .ClusterName}}{{- end -}}"
-  # Unique ID of the cluster. Must be unique across all conneted clusters and
+
-  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
-  # DNS Polling periodically issues a DNS lookup for each `matchName` from
-  # cilium-agent. The result is used to regenerate endpoint policy.
-  # DNS lookups are repeated with an interval of 5 seconds, and are made for
-  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
-  # data is used instead. An IP change will trigger a regeneration of the Cilium
-  # policy for each endpoint and increment the per cilium-agent policy
-  # repository revision.
-  #
   # This option is disabled by default starting from version 1.4.x in favor
   # of a more powerful DNS proxy-based implementation, see [0] for details.
   # Enable this option if you want to use FQDN policies but do not want to use
@@ -89,9 +83,9 @@ data:
   #
   # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
   # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
-  tofqdns-enable-poller: "false"
+  tofqdns-enable-poller: "{{- if .ToFqdnsEnablePoller -}}true{{- else -}}false{{- end -}}"
   # wait-bpf-mount makes init container wait until bpf filesystem is mounted
-  wait-bpf-mount: "false"
+  wait-bpf-mount: "true"
   # Enable fetching of container-runtime specific metadata
   #
   # By default, the Kubernetes pod and namespace labels are retrieved and
@@ -111,11 +105,12 @@ data:
   # - none
   # - auto (automatically detect the container runtime)
   #
-  container-runtime: none
+  container-runtime: "{{- if eq .ContainerRuntimeLabels "" -}}none{{- else -}}{{ .ContainerRuntimeLabels }}{{- end -}}"
-  masquerade: "true"
+  masquerade: "{{- if .DisableMasquerade -}}false{{- else -}}true{{- end -}}"
-  install-iptables-rules: "true"
+  install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
-  auto-direct-node-routes: "false"
+  auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
-  enable-node-port: "false"
+  enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
+{{ end }} # With .Networking.Cilium end
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -622,3 +617,110 @@ spec:
       restartPolicy: Always
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: cilium-node-init
+  namespace: kube-system
+  labels:
+    app: cilium-node-init
+spec:
+  selector:
+    matchLabels:
+      app: cilium-node-init
+  template:
+    metadata:
+      labels:
+        app: cilium-node-init
+    spec:
+      tolerations:
+      - operator: Exists
+      hostPID: true
+      hostNetwork: true
+      containers:
+        - name: node-init
+          image: gcr.io/google-containers/startup-script:v1
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: true
+          env:
+          # STARTUP_SCRIPT is the script run on node bootstrap. Node
+          # bootstrapping can be customized in this script.
+          - name: STARTUP_SCRIPT
+            value: |
+              #!/bin/bash
+              set -o errexit
+              set -o pipefail
+              set -o nounset
+              mount | grep "/sys/fs/bpf type bpf" || {
+                # Mount the filesystem until next reboot
+                echo "Mounting BPF filesystem..."
+                mount bpffs /sys/fs/bpf -t bpf
+              }
+              echo "Installing BPF filesystem mount"
+              cat >/tmp/sys-fs-bpf.mount <<EOF
+              [Unit]
+              Description=Mount BPF filesystem (Cilium)
+              Documentation=http://docs.cilium.io/
+              DefaultDependencies=no
+              Before=local-fs.target umount.target
+              After=swap.target
+              [Mount]
+              What=bpffs
+              Where=/sys/fs/bpf
+              Type=bpf
+              [Install]
+              WantedBy=multi-user.target
+              EOF
+              if [ -d "/etc/systemd/system/" ]; then
+                mv /tmp/sys-fs-bpf.mount /etc/systemd/system/
+                echo "Installed sys-fs-bpf.mount to /etc/systemd/system/"
+              elif [ -d "/lib/systemd/system/" ]; then
+                mv /tmp/sys-fs-bpf.mount /lib/systemd/system/
+                echo "Installed sys-fs-bpf.mount to /lib/systemd/system/"
+              fi
+              # Azure specific:
+              if [ -f /var/run/azure-vnet.json ]; then
+                sed -i 's/"Mode": "bridge",/"Mode": "transparent",/g' /var/run/azure-vnet.json
+              fi
+              # Ensure that filesystem gets mounted on next reboot
+              systemctl enable sys-fs-bpf.mount
+              systemctl start sys-fs-bpf.mount
+              echo "Link information:"
+              ip link
+              echo "Routing table:"
+              ip route
+              echo "Addressing:"
+              ip -4 a
+              ip -6 a
+              {{ with .Networking.Cilium }}
+              {{ if .RemoveCbrBridge }}
+              if ip link show cbr0; then
+                echo "Detected cbr0 bridge. Deleting interface..."
+                ip link del cbr0
+              fi
+              {{ end }}
+              {{ if .RestartPods }}
+              echo "Restarting kubenet managed pods"
+              if grep -q 'docker' /etc/crictl.yaml; then
+                # Works for COS, ubuntu
+                for f in `find /var/lib/cni/networks/ -type f ! -name lock ! -name last_reserved_ip.0`; do docker rm -f $(cat $f) || true; done
+              else
+                # COS-beta (with containerd)
+                for f in `find /var/lib/cni/networks/ -type f ! -name lock ! -name last_reserved_ip.0`; do crictl stopp $(cat $f) || true; done
+              fi
+              {{ end }}
+              {{ if .ReconfigureKubelet }}
+              # GKE: Alter the kubelet configuration to run in CNI mode
+              echo "Changing kubelet configuration to --network-plugin=cni --cni-bin-dir={{ .CniBinPath }}"
+              mkdir -p {{ .CniBinPath }}
+              sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir={{ .CniBinPath }}:g" /etc/default/kubelet
+              echo "Restarting kubelet..."
+              systemctl restart kubelet
+              {{ end }}
+              {{ if not (eq .NodeInitBootstrapFile "") }}
+              date > {{ .Values.global.nodeinit.bootstrapFile }}
+              {{ end }}
+              {{ end }}#end of with .Networking.Cilium
+              echo "Node initialization complete"

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -107,7 +107,7 @@ spec:
   - id: k8s-1.7
     kubernetesVersion: '>=1.7.0 <1.12.0'
     manifest: networking.cilium.io/k8s-1.7.yaml
-    manifestHash: 26096db7dfad3f26c8b2fc92cd619d7dbc8c8ecd
+    manifestHash: a66bde86dd1e1ef7e2f16f828c7520d2b1a2fa91
     name: networking.cilium.io
     selector:
       role.kubernetes.io/networking: "1"
@@ -115,7 +115,7 @@ spec:
   - id: k8s-1.12
     kubernetesVersion: '>=1.12.0'
     manifest: networking.cilium.io/k8s-1.12.yaml
-    manifestHash: e4886cb88b110e5509929088f83b6d23cf1bbaa0
+    manifestHash: a66bde86dd1e1ef7e2f16f828c7520d2b1a2fa91
     name: networking.cilium.io
     selector:
       role.kubernetes.io/networking: "1"