{{ with .Networking.Cilium }} {{ $semver := (trimPrefix "v" .Version) }} {{ $healthPort := (ternary 9879 9876 (semverCompare ">=1.11.6" $semver)) }} {{ $operatorHealthPort := 9234 }} {{- if CiliumSecret }} apiVersion: v1 kind: Secret metadata: name: cilium-ipsec-keys namespace: kube-system stringData: {{ CiliumSecret }} --- {{- end }} --- apiVersion: v1 kind: ServiceAccount metadata: name: cilium namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: cilium-operator namespace: kube-system {{ if WithDefaultBool .Hubble.Enabled false }} --- apiVersion: v1 kind: ServiceAccount metadata: name: hubble-relay namespace: kube-system {{ end }} --- apiVersion: v1 kind: ConfigMap metadata: name: cilium-config namespace: kube-system data: agent-health-port: "{{ $healthPort }}" {{- if .EtcdManaged }} kvstore: etcd kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}' etcd-config: |- --- endpoints: - https://{{ APIInternalName }}:4003 trusted-ca-file: '/var/lib/etcd-secrets/etcd-ca.crt' key-file: '/var/lib/etcd-secrets/etcd-client-cilium.key' cert-file: '/var/lib/etcd-secrets/etcd-client-cilium.crt' enable-k8s-event-handover: "true" {{ end }} # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). # These can be queried with: # kubectl get ciliumid # - "kvstore" stores identities in a kvstore, etcd or consul, that is # configured below. Cilium versions before 1.6 supported only the kvstore # backend. Upgrades from these older cilium versions should continue using # the kvstore by commenting out the identity-allocation-mode below, or # setting it to "kvstore". # (default crd) identity-allocation-mode: "{{ .IdentityAllocationMode }}" # Time to wait before using new identity on endpoint identity change (default 5s) identity-change-grace-period: "{{ .IdentityChangeGracePeriod }}" # If you want to run cilium in debug mode change this value to true debug: "{{ .Debug }}" {{ if .EnablePrometheusMetrics }} # If you want metrics enabled in all of your Cilium agents, set the port for # which the Cilium agents will have their metrics exposed. # This option deprecates the "prometheus-serve-addr" in the # "cilium-metrics-config" ConfigMap # NOTE that this will open the port on ALL nodes where Cilium pods are # scheduled. prometheus-serve-addr: ":{{ .AgentPrometheusPort }}" operator-prometheus-serve-addr: ":6942" enable-metrics: "true" {{ end }} {{ if .Metrics }} # Metrics that should be enabled or disabled from the default metric # list. (+metric_foo to enable metric_foo , -metric_bar to disable # metric_bar). metrics: {{- range .Metrics }} {{ . }} {{- end }} {{ end }} {{ if .EnableEncryption }} {{ if eq .EncryptionType "ipsec" }} enable-ipsec: "true" ipsec-key-file: /etc/ipsec/keys {{ else if eq .EncryptionType "wireguard" }} enable-wireguard: "true" {{ end }} encrypt-node: "{{ .NodeEncryption }}" {{ end }} # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "{{ not IsIPv6Only }}" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. enable-ipv6: "{{ IsIPv6Only }}" # If you want cilium monitor to aggregate tracing for packets, set this level # to "low", "medium", or "maximum". The higher the level, the less packets # that will be seen in monitor output. monitor-aggregation: "{{ .MonitorAggregation }}" # ct-global-max-entries-* specifies the maximum number of connections # supported across all endpoints, split by protocol: tcp or other. One pair # of maps uses these values for IPv4 connections, and another pair of maps # use these values for IPv6 connections. # # If these values are modified, then during the next Cilium startup the # tracking of ongoing connections may be disrupted. This may lead to brief # policy drops or a change in loadbalancing decisions for a connection. # # For users upgrading from Cilium 1.2 or earlier, to minimize disruption # during the upgrade process, comment out these options. bpf-ct-global-tcp-max: "{{ .BPFCTGlobalTCPMax }}" bpf-ct-global-any-max: "{{ .BPFCTGlobalAnyMax }}" # BPF load balancing algorithm ("random", "maglev") (default "random") bpf-lb-algorithm: "{{ .BPFLBAlgorithm }}" # Maglev per service backend table size (parameter M) (default 16381) bpf-lb-maglev-table-size: "{{ .BPFLBMaglevTableSize }}" # bpf-nat-global-max specified the maximum number of entries in the # BPF NAT table. (default 524288) bpf-nat-global-max: "{{ .BPFNATGlobalMax }}" # bpf-neigh-global-max specified the maximum number of entries in the # BPF neighbor table. (default 524288) bpf-neigh-global-max: "{{ .BPFNeighGlobalMax }}" # bpf-policy-map-max specifies the maximum number of entries in endpoint # policy map (per endpoint) (default 16384) bpf-policy-map-max: "{{ .BPFPolicyMapMax }}" # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, # backend and affinity maps. (default 65536) bpf-lb-map-max: "{{ .BPFLBMapMax }}" # bpf-lb-sock-hostns-only enables skipping socket LB for services when inside a pod namespace, # in favor of service LB at the pod interface. Socket LB is still used when in the host namespace. # Required by service mesh (e.g., Istio, Linkerd). (default false) bpf-lb-sock-hostns-only: "{{ .BPFLBSockHostNSOnly }}" {{ if .ChainingMode }} cni-chaining-mode: "{{ .ChainingMode }}" {{ end }} # enable-bpf-masquerade enables masquerading packets from endpoints leaving # the host with BPF instead of iptables. (default false) enable-bpf-masquerade: "{{ and (WithDefaultBool .EnableBPFMasquerade false) (not IsIPv6Only) }}" # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; # users who are sensitive to latency may consider setting this to "true". # # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore # this option and behave as though it is set to "true". # # If this value is modified, then during the next Cilium startup the restore # of existing endpoints and tracking of ongoing connections may be disrupted. # This may lead to policy drops or a change in loadbalancing decisions for a # connection for some time. Endpoints may need to be recreated to restore # connectivity. # # If this option is set to "false" during an upgrade from 1.3 or earlier to # 1.4 or later, then it may cause one-time disruptions during the upgrade. preallocate-bpf-maps: "{{- if .PreallocateBPFMaps -}}true{{- else -}}false{{- end -}}" # Regular expression matching compatible Istio sidecar istio-proxy # container image names sidecar-istio-proxy-image: "{{ .SidecarIstioProxyImage }}" # Encapsulation mode for communication between nodes # Possible values: # - disabled # - vxlan (default) # - geneve {{ if eq .Tunnel "disabled" }} # This option enables native-routing mode, in place of tunnel=disabled, now deprecated. routing-mode: "native" {{ else }} routing-mode: "tunnel" tunnel-protocol: "{{ .Tunnel }}" {{ end }} # Name of the cluster. Only relevant when building a mesh of clusters. cluster-name: "{{ .ClusterName }}" # ID of the cluster. Only relevant when building a mesh of clusters. # Must be a number between 1 and 255. {{ if .ClusterID }} cluster-id: "{{ .ClusterID }}" {{ end }} remove-cilium-node-taints: "true" set-cilium-node-taints: "true" set-cilium-is-up-condition: "true" # DNS response code for rejecting DNS requests, # available options are "nameError" and "refused" tofqdns-dns-reject-response-code: "{{ .ToFQDNsDNSRejectResponseCode }}" # This option is disabled by default starting from version 1.4.x in favor # of a more powerful DNS proxy-based implementation, see [0] for details. # Enable this option if you want to use FQDN policies but do not want to use # the DNS proxy. # # To ease upgrade, users may opt to set this option to "true". # Otherwise please refer to the Upgrade Guide [1] which explains how to # prepare policy rules for upgrade. # # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action tofqdns-enable-poller: "{{- if .ToFQDNsEnablePoller -}}true{{- else -}}false{{- end -}}" {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }} # wait-bpf-mount makes init container wait until bpf filesystem is mounted wait-bpf-mount: "false" {{- end }} # Enable fetching of container-runtime specific metadata # # By default, the Kubernetes pod and namespace labels are retrieved and # associated with endpoints for identification purposes. By integrating # with the container runtime, container runtime specific labels can be # retrieved, such labels will be prefixed with container: # # CAUTION: The container runtime labels can include information such as pod # annotations which may result in each pod being associated a unique set of # labels which can result in excessive security identities being allocated. # Please review the labels filter when enabling container runtime labels. # # Supported values: # - containerd # - crio # - docker # - none # - auto (automatically detect the container runtime) # enable-ipv4-masquerade: "{{ .Masquerade }}" enable-ipv6-masquerade: "false" install-iptables-rules: "{{ WithDefaultBool .InstallIptablesRules true }}" auto-direct-node-routes: "{{ .AutoDirectNodeRoutes }}" {{ if .EnableHostReachableServices }} enable-host-reachable-services: "{{ .EnableHostReachableServices }}" {{ end }} enable-node-port: "{{ .EnableNodePort }}" kube-proxy-replacement: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}" {{ with .IPAM }} ipam: {{ . }} {{ if eq . "eni" }} enable-endpoint-routes: "true" auto-create-cilium-node-resource: "true" blacklist-conflicting-routes: "false" {{ end }} {{ end }} # Disables usage of CiliumEndpoint CRD disable-endpoint-crd: "{{ .DisableEndpointCRD }}" # Enable connectivity health checking between virtual endpoints (default true) enable-endpoint-health-checking: "{{ .EnableEndpointHealthChecking }}" # Enable use of remote node identity (default false) enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}" # Enable unreachable routes on pod deletion (default false) enable-unreachable-routes: "{{ .EnableUnreachableRoutes }}" # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true) enable-l7-proxy: "{{ .EnableL7Proxy }}" cgroup-root: /run/cilium/cgroupv2 disable-cnp-status-updates: "{{ .DisableCNPStatusUpdates }}" nodes-gc-interval: "5m0s" enable-service-topology: "{{ .EnableServiceTopology }}" {{ if WithDefaultBool .Ingress.Enabled false }} enable-envoy-config: "true" external-envoy-proxy: "false" enable-ingress-controller: "true" ingress-secrets-namespace: kube-system {{ if .Ingress.EnforceHttps }} enforce-ingress-https: "{{ .Ingress.EnforceHttps }}" {{ end }} {{ if .Ingress.EnableSecretsSync }} enable-ingress-secrets-sync: "{{ .Ingress.EnableSecretsSync }}" {{ end }} {{ if .Ingress.SharedLoadBalancerServiceName }} ingress-shared-lb-service-name: {{ .Ingress.SharedLoadBalancerServiceName }} {{ end }} {{ if .Ingress.DefaultLoadBalancerMode }} ingress-default-lb-mode: {{ .Ingress.DefaultLoadBalancerMode }} {{ end }} {{ if .Ingress.LoadBalancerAnnotationPrefixes }} ingress-lb-annotation-prefixes: "{{ .Ingress.LoadBalancerAnnotationPrefixes }}" {{ end }} {{ end }} # Tell the agent to generate and write a CNI configuration file write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist cni-exclusive: "true" cni-log-file: "/var/run/cilium/cilium-cni.log" {{ if WithDefaultBool .Hubble.Enabled false }} # Enable Hubble gRPC service. enable-hubble: "true" # UNIX domain socket for Hubble server to listen to. hubble-socket-path: "/var/run/cilium/hubble.sock" # An additional address for Hubble server to listen to (e.g. ":4244"). hubble-listen-address: ":4244" hubble-disable-tls: "false" hubble-tls-cert-file: /var/lib/cilium/tls/hubble/tls.crt hubble-tls-key-file: /var/lib/cilium/tls/hubble/tls.key hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/ca.crt {{ if .Hubble.Metrics }} hubble-metrics-server: ":9091" hubble-metrics: {{- range .Hubble.Metrics }} {{ . }} {{- end }} {{ end }} {{ end }} {{ if WithDefaultBool .Hubble.Enabled false }} --- # Source: cilium/templates/hubble-relay-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: hubble-relay-config namespace: kube-system data: config.yaml: | peer-service: unix:///var/run/cilium/hubble.sock listen-address: :4245 disable-server-tls: true tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt tls-client-key-file: /var/lib/hubble-relay/tls/client.key tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt {{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium rules: - apiGroups: - networking.k8s.io resources: - networkpolicies verbs: - get - list - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: - namespaces - services - pods - endpoints - nodes verbs: - get - list - watch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - list - watch - get - apiGroups: - cilium.io resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies - ciliumendpoints - ciliumendpointslices - ciliumenvoyconfigs - ciliumidentities - ciliumlocalredirectpolicies - ciliumnetworkpolicies - ciliumnodes - ciliumnodeconfigs - ciliumcidrgroups - ciliuml2announcementpolicies - ciliumpodippools verbs: - list - watch - apiGroups: - cilium.io resources: - ciliumidentities - ciliumendpoints - ciliumnodes verbs: - create - apiGroups: - cilium.io # To synchronize garbage collection of such resources resources: - ciliumidentities verbs: - update - apiGroups: - cilium.io resources: - ciliumendpoints verbs: - delete - get - apiGroups: - cilium.io resources: - ciliumnodes - ciliumnodes/status verbs: - get - update - apiGroups: - cilium.io resources: - ciliumnetworkpolicies/status - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status verbs: - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-operator rules: - apiGroups: - "" resources: - pods verbs: - get - list - watch # to automatically delete [core|kube]dns pods so that are starting to being # managed by Cilium - delete - apiGroups: - "" resources: - nodes verbs: - list - watch - apiGroups: - "" resources: # To remove node taints - nodes # To set NetworkUnavailable false on startup - nodes/status verbs: - patch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: # to perform LB IP allocation for BGP - services/status verbs: - update - patch - apiGroups: - "" resources: # to check apiserver connectivity - namespaces verbs: - get - list - watch - apiGroups: - "" resources: # to perform the translation of a CNP that contains `ToGroup` to its endpoints - services - endpoints verbs: - get - list - watch - apiGroups: - cilium.io resources: - ciliumnetworkpolicies - ciliumclusterwidenetworkpolicies verbs: # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - create - update - deletecollection # To update the status of the CNPs and CCNPs - patch - get - list - watch - apiGroups: - cilium.io resources: - ciliumnetworkpolicies/status - ciliumclusterwidenetworkpolicies/status verbs: # Update the auto-generated CNPs and CCNPs status. - patch - update - apiGroups: - cilium.io resources: - ciliumendpoints - ciliumidentities verbs: # To perform garbage collection of such resources - delete - list - watch - apiGroups: - cilium.io resources: - ciliumidentities verbs: # To synchronize garbage collection of such resources - update - apiGroups: - cilium.io resources: - ciliumnodes verbs: - create - update - get - list - watch # To perform CiliumNode garbage collector - delete - apiGroups: - cilium.io resources: - ciliumnodes/status verbs: - update - apiGroups: - cilium.io resources: - ciliumendpointslices - ciliumenvoyconfigs verbs: - create - update - get - list - watch - delete - patch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - create - get - list - watch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - update resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io - ciliumendpoints.cilium.io - ciliumendpointslices.cilium.io - ciliumenvoyconfigs.cilium.io - ciliumexternalworkloads.cilium.io - ciliumidentities.cilium.io - ciliumlocalredirectpolicies.cilium.io - ciliumnetworkpolicies.cilium.io - ciliumnodes.cilium.io - ciliumnodeconfigs.cilium.io - ciliumcidrgroups.cilium.io - ciliuml2announcementpolicies.cilium.io - ciliumpodippools.cilium.io - apiGroups: - cilium.io resources: - ciliumloadbalancerippools - ciliumpodippools verbs: - get - list - watch - apiGroups: - cilium.io resources: - ciliumpodippools verbs: - create - apiGroups: - cilium.io resources: - ciliumloadbalancerippools/status verbs: - patch - apiGroups: - coordination.k8s.io resources: - leases verbs: - create - get - update {{ if WithDefaultBool .Ingress.Enabled false }} - apiGroups: - networking.k8s.io resources: - ingresses - ingressclasses verbs: - get - list - watch - apiGroups: - networking.k8s.io resources: - ingresses/status # To update ingress status with load balancer IP. verbs: - update {{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium subjects: - kind: ServiceAccount name: cilium namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium-operator subjects: - kind: ServiceAccount name: cilium-operator namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-config-agent namespace: kube-system labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - configmaps verbs: - get - list - watch --- # Source: cilium/templates/cilium-agent/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-config-agent namespace: kube-system labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-config-agent subjects: - kind: ServiceAccount name: "cilium" namespace: kube-system {{ if WithDefaultBool .Ingress.Enabled false }} --- # Source: cilium/templates/cilium-agent/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-ingress-secrets namespace: kube-system labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - secrets verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-secrets namespace: kube-system labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-ingress-secrets subjects: - kind: ServiceAccount name: "cilium" namespace: kube-system --- # Source: cilium/templates/cilium-operator/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-operator-ingress-secrets namespace: kube-system labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - secrets verbs: - create - delete - update - patch --- # Source: cilium/templates/cilium-operator/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-operator-ingress-secrets namespace: kube-system labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-operator-ingress-secrets subjects: - kind: ServiceAccount name: "cilium-operator" namespace: kube-system --- # Source: cilium/templates/cilium-ingress-class.yaml apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: name: cilium spec: controller: cilium.io/ingress-controller {{ if or (eq .Ingress.DefaultLoadBalancerMode "shared") (not .Ingress.DefaultLoadBalancerMode) }} --- # Source: cilium/templates/cilium-ingress-service.yaml apiVersion: v1 kind: Service metadata: name: {{ .Ingress.SharedLoadBalancerServiceName }} namespace: kube-system labels: cilium.io/ingress: "true" spec: ports: - name: http port: 80 protocol: TCP nodePort: - name: https port: 443 protocol: TCP nodePort: type: LoadBalancer --- # Source: cilium/templates/cilium-ingress-service.yaml apiVersion: v1 kind: Endpoints metadata: name: {{ .Ingress.SharedLoadBalancerServiceName }} namespace: kube-system subsets: - addresses: - ip: "192.192.192.192" ports: - port: 9999 {{ end }} {{ end }} {{ if WithDefaultBool .Hubble.Enabled false }} --- # Source: cilium/templates/hubble-relay-service.yaml kind: Service apiVersion: v1 metadata: name: hubble-relay namespace: kube-system labels: k8s-app: hubble-relay spec: type: ClusterIP selector: k8s-app: hubble-relay ports: - protocol: TCP port: 80 targetPort: 4245 {{ end }} --- apiVersion: apps/v1 kind: DaemonSet metadata: name: cilium namespace: kube-system labels: k8s-app: cilium kubernetes.io/cluster-service: "true" spec: selector: matchLabels: k8s-app: cilium kubernetes.io/cluster-service: "true" updateStrategy: rollingUpdate: maxUnavailable: 2 type: RollingUpdate template: metadata: annotations: {{ if .EnablePrometheusMetrics }} # Annotation required for prometheus auto-discovery scraping # https://docs.cilium.io/en/v1.9/operations/metrics/#installation prometheus.io/scrape: "true" prometheus.io/port: "{{ .AgentPrometheusPort }}" {{ end }} {{- range $key, $value := .AgentPodAnnotations }} {{ $key }}: "{{ $value }}" {{- end }} labels: k8s-app: cilium kubernetes.io/cluster-service: "true" spec: containers: - name: cilium-agent image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - cilium-agent args: - --config-dir=/tmp/cilium/config-map startupProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz port: {{ $healthPort }} scheme: HTTP httpHeaders: - name: "brief" value: "true" failureThreshold: 105 periodSeconds: 2 successThreshold: 1 livenessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz port: {{ $healthPort }} scheme: HTTP httpHeaders: - name: "brief" value: "true" periodSeconds: 30 successThreshold: 1 failureThreshold: 10 timeoutSeconds: 5 resources: requests: cpu: {{ or .CPURequest "25m" }} memory: {{ or .MemoryRequest "128Mi" }} readinessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz port: {{ $healthPort }} scheme: HTTP httpHeaders: - name: "brief" value: "true" periodSeconds: 30 successThreshold: 1 failureThreshold: 3 timeoutSeconds: 5 env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - name: CILIUM_CNI_CHAINING_MODE valueFrom: configMapKeyRef: key: cni-chaining-mode name: cilium-config optional: true - name: CILIUM_CUSTOM_CNI_CONF valueFrom: configMapKeyRef: key: custom-cni-conf name: cilium-config optional: true - name: KUBERNETES_SERVICE_HOST value: "{{ APIInternalName }}" - name: KUBERNETES_SERVICE_PORT value: "443" {{ with .EnablePolicy }} - name: CILIUM_ENABLE_POLICY value: {{ . }} {{ end }} lifecycle: {{ if eq .IPAM "eni" }} postStart: exec: command: - "bash" - "-c" - | set -o errexit set -o pipefail set -o nounset # When running in AWS ENI mode, it's likely that 'aws-node' has # had a chance to install SNAT iptables rules. These can result # in dropped traffic, so we should attempt to remove them. # We do it using a 'postStart' hook since this may need to run # for nodes which might have already been init'ed but may still # have dangling rules. This is safe because there are no # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). if [[ "$(iptables-save | grep -c AWS-SNAT-CHAIN)" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' iptables-save | grep -v AWS-SNAT-CHAIN | iptables-restore fi echo 'Done!' {{- end }} preStop: exec: command: - /cni-uninstall.sh ports: {{ if .EnablePrometheusMetrics }} - containerPort: {{ .AgentPrometheusPort }} name: prometheus protocol: TCP {{ end }} {{- if .Hubble.Metrics }} - containerPort: 9091 hostPort: 9091 name: hubble-metrics protocol: TCP {{- end }} terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf {{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }} mountPropagation: Bidirectional {{- end }} - name: cilium-cgroup mountPath: /run/cilium/cgroupv2 - name: cilium-run mountPath: /var/run/cilium {{- if not (semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver) }} - name: cni-path mountPath: /host/opt/cni/bin {{- end }} - name: etc-cni-netd mountPath: /host/etc/cni/net.d {{ if .EtcdManaged }} - name: etcd-config-path mountPath: /var/lib/etcd-config readOnly: true - name: etcd-secrets mountPath: /var/lib/etcd-secrets readOnly: true {{ end }} - name: clustermesh-secrets mountPath: /var/lib/cilium/clustermesh readOnly: true - name: cilium-config-path mountPath: /tmp/cilium/config-map readOnly: true # Needed to be able to load kernel modules - name: lib-modules mountPath: /lib/modules readOnly: true - name: xtables-lock mountPath: /run/xtables.lock - name: tmp mountPath: /tmp {{ if WithDefaultBool .Hubble.Enabled false }} - name: hubble-tls mountPath: /var/lib/cilium/tls/hubble readOnly: true {{ end }} {{ if CiliumSecret }} - mountPath: /etc/ipsec name: cilium-ipsec-secrets {{ end }} {{ if .Debug }} - name: cilium-monitor image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - /bin/bash - -c - -- args: - |- for i in {1..5}; do \ [ -S /var/run/cilium/monitor1_2.sock ] && break || sleep 10;\ done; \ cilium monitor --type=agent terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cilium-run mountPath: /var/run/cilium {{ end }} initContainers: - name: config image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - cilium - build-config env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: KUBERNETES_SERVICE_HOST value: "{{ APIInternalName }}" - name: KUBERNETES_SERVICE_PORT value: "443" volumeMounts: - name: tmp mountPath: /tmp terminationMessagePolicy: FallbackToLogsOnError # Required to mount cgroup2 filesystem on the underlying Kubernetes node. # We use nsenter command with host's cgroup and mount namespaces enabled. - name: mount-cgroup image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent env: - name: CGROUP_ROOT value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin command: - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh and mount that can be missing on certain # distros installed on the underlying host. Copy the binary to the # same directory where we install cilium cni plugin so that exec permissions # are available. - | cp /usr/bin/cilium-mount /hostbin/cilium-mount; nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; rm /hostbin/cilium-mount volumeMounts: - name: hostproc mountPath: /hostproc - name: cni-path mountPath: /hostbin terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true - name: apply-sysctl-overwrites image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent env: - name: BIN_PATH value: /opt/cni/bin command: - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh that can be missing on certain # distros installed on the underlying host. Copy the binary to the # same directory where we install cilium cni plugin so that exec permissions # are available. - | cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; rm /hostbin/cilium-sysctlfix volumeMounts: - name: hostproc mountPath: /hostproc - name: cni-path mountPath: /hostbin terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true - name: clean-cilium-state image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - /init-container.sh env: - name: CILIUM_ALL_STATE valueFrom: configMapKeyRef: name: cilium-config key: clean-cilium-state optional: true - name: CILIUM_BPF_STATE valueFrom: configMapKeyRef: name: cilium-config key: clean-cilium-bpf-state optional: true - name: KUBERNETES_SERVICE_HOST value: "{{ APIInternalName }}" - name: KUBERNETES_SERVICE_PORT value: "443" {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }} - name: CILIUM_WAIT_BPF_MOUNT valueFrom: configMapKeyRef: key: wait-bpf-mount name: cilium-config optional: true {{- end }} terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf {{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }} mountPropagation: HostToContainer {{- end }} # Required to mount cgroup filesystem from the host to cilium agent pod - name: cilium-cgroup mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer - name: cilium-run mountPath: /var/run/cilium {{- if semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver }} # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - /install-plugin.sh resources: requests: cpu: 100m memory: 10Mi securityContext: capabilities: drop: - ALL terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cni-path mountPath: /host/opt/cni/bin {{- end }} restartPolicy: Always priorityClassName: system-node-critical {{ if ContainerdSELinuxEnabled }} securityContext: seLinuxOptions: type: spc_t level: s0 {{ end }} serviceAccount: cilium serviceAccountName: cilium terminationGracePeriodSeconds: 1 hostNetwork: true affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: k8s-app: cilium topologyKey: kubernetes.io/hostname nodeSelector: kubernetes.io/os: linux tolerations: - operator: Exists volumes: # For sharing configuration between the "config" initContainer and the agent - name: tmp emptyDir: {} # To keep state between restarts / upgrades - name: cilium-run hostPath: path: /var/run/cilium type: DirectoryOrCreate # To keep state between restarts / upgrades for bpf maps - name: bpf-maps hostPath: path: /sys/fs/bpf type: DirectoryOrCreate # To mount cgroup2 filesystem on the host - name: hostproc hostPath: path: /proc type: Directory # To keep state between restarts / upgrades for cgroup2 filesystem - name: cilium-cgroup hostPath: path: /run/cilium/cgroupv2 type: DirectoryOrCreate # To install cilium cni plugin in the host - name: cni-path hostPath: path: /opt/cni/bin type: DirectoryOrCreate # To install cilium cni configuration in the host - name: etc-cni-netd hostPath: path: /etc/cni/net.d type: DirectoryOrCreate # To be able to load kernel modules - name: lib-modules hostPath: path: /lib/modules # To access iptables concurrently with other processes (e.g. kube-proxy) - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate {{- if .EtcdManaged }} # To read the etcd config stored in config maps - name: etcd-config-path configMap: name: cilium-config # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 items: - key: etcd-config path: etcd.config # To read the Cilium etcd secrets in case the user might want to use TLS - name: etcd-secrets hostPath: path: /etc/kubernetes/pki/cilium type: Directory {{- end }} - name: clustermesh-secrets projected: # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 sources: - secret: name: cilium-clustermesh optional: true # note: items are not explicitly listed here, since the entries of this secret # depend on the peers configured, and that would cause a restart of all agents # at every addition/removal. Leaving the field empty makes each secret entry # to be automatically projected into the volume as a file whose name is the key. - secret: name: clustermesh-apiserver-remote-cert optional: true items: - key: tls.key path: common-etcd-client.key - key: tls.crt path: common-etcd-client.crt - key: ca.crt path: common-etcd-client-ca.crt - configMap: name: cilium-config name: cilium-config-path {{ if CiliumSecret }} - name: cilium-ipsec-secrets secret: secretName: cilium-ipsec-keys {{ end }} {{ if WithDefaultBool .Hubble.Enabled false }} - name: hubble-tls projected: # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 sources: - secret: name: hubble-server-certs optional: true {{ end }} --- apiVersion: apps/v1 kind: Deployment metadata: labels: io.cilium/app: operator name: cilium-operator name: cilium-operator namespace: kube-system spec: replicas: {{ ControlPlaneControllerReplicas false }} selector: matchLabels: io.cilium/app: operator name: cilium-operator strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: metadata: annotations: {{- range $key, $value := .OperatorPodAnnotations }} {{ $key }}: "{{ $value }}" {{- end }} labels: io.cilium/app: operator name: cilium-operator spec: nodeSelector: null affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists containers: - name: cilium-operator image: "{{ or .Registry "quay.io" }}/cilium/operator:{{ .Version }}" imagePullPolicy: IfNotPresent command: - cilium-operator args: - "--config-dir=/tmp/cilium/config-map" - "--debug=$(CILIUM_DEBUG)" - "--eni-tags={{ CloudLabels }}" env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_DEBUG valueFrom: configMapKeyRef: key: debug name: cilium-config optional: true - name: KUBERNETES_SERVICE_HOST value: "{{ APIInternalName }}" - name: KUBERNETES_SERVICE_PORT value: "443" {{ if .EnablePrometheusMetrics }} ports: - containerPort: 6942 hostPort: 6942 name: prometheus protocol: TCP {{ end }} resources: requests: cpu: {{ or .CPURequest "25m" }} memory: {{ or .MemoryRequest "128Mi" }} livenessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz port: {{ $operatorHealthPort }} scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 readinessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz port: {{ $operatorHealthPort }} scheme: HTTP initialDelaySeconds: 0 periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 5 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /tmp/cilium/config-map name: cilium-config-path readOnly: true {{- if .EtcdManaged }} - mountPath: /var/lib/etcd-config name: etcd-config-path readOnly: true - mountPath: /var/lib/etcd-secrets name: etcd-secrets readOnly: true {{- end }} hostNetwork: true restartPolicy: Always priorityClassName: system-cluster-critical {{ if ContainerdSELinuxEnabled }} securityContext: seLinuxOptions: type: spc_t level: s0 {{ end }} serviceAccount: cilium-operator serviceAccountName: cilium-operator tolerations: - operator: Exists topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: io.cilium/app: operator name: cilium-operator - maxSkew: 1 topologyKey: "kubernetes.io/hostname" whenUnsatisfiable: DoNotSchedule labelSelector: matchLabels: io.cilium/app: operator name: cilium-operator volumes: # To read the configuration from the config map - configMap: name: cilium-config name: cilium-config-path {{- if .EtcdManaged }} # To read the etcd config stored in config maps - configMap: defaultMode: 420 items: - key: etcd-config path: etcd.config name: cilium-config name: etcd-config-path # To read the k8s etcd secrets in case the user might want to use TLS - name: etcd-secrets hostPath: path: /etc/kubernetes/pki/cilium type: Directory {{- end }} {{ if WithDefaultBool .Hubble.Enabled false }} --- # Source: cilium/charts/hubble-relay/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: hubble-relay namespace: kube-system labels: k8s-app: hubble-relay spec: replicas: 2 selector: matchLabels: k8s-app: hubble-relay strategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate template: metadata: labels: k8s-app: hubble-relay spec: securityContext: fsGroup: 65532 containers: - name: hubble-relay image: "{{ or .Registry "quay.io" }}/cilium/hubble-relay:{{ .Version }}" imagePullPolicy: IfNotPresent securityContext: capabilities: drop: - ALL runAsGroup: 65532 runAsNonRoot: true runAsUser: 65532 command: - hubble-relay args: - "serve" - "--peer-service=unix:///var/run/cilium/hubble.sock" - "--listen-address=:4245" env: # unfortunately, the addon CAs use only CN - name: GODEBUG value: x509ignoreCN=0 ports: - name: grpc containerPort: 4245 readinessProbe: tcpSocket: port: grpc livenessProbe: tcpSocket: port: grpc terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/run/cilium name: hubble-sock-dir readOnly: true - mountPath: /etc/hubble-relay name: config readOnly: true - mountPath: /var/lib/hubble-relay/tls name: tls readOnly: true restartPolicy: Always serviceAccount: hubble-relay serviceAccountName: hubble-relay terminationGracePeriodSeconds: 0 topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: k8s-app: hubble-relay - maxSkew: 1 topologyKey: "kubernetes.io/hostname" whenUnsatisfiable: DoNotSchedule labelSelector: matchLabels: k8s-app: hubble-relay volumes: - hostPath: path: /var/run/cilium type: Directory name: hubble-sock-dir - configMap: name: hubble-relay-config items: - key: config.yaml path: config.yaml name: config - projected: sources: - secret: name: hubble-relay-client-certs items: - key: tls.crt path: client.crt - key: tls.key path: client.key - key: ca.crt path: hubble-server-ca.crt name: tls --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: k8s-app: cilium name: hubble-server-certs namespace: kube-system spec: dnsNames: - "*.{{ replace .ClusterName "." "-" }}.hubble-grpc.cilium.io" issuerRef: kind: Issuer name: networking.cilium.io secretName: hubble-server-certs --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: k8s-app: cilium name: hubble-relay-client-certs namespace: kube-system spec: dnsNames: - "hubble-relay-client" issuerRef: kind: Issuer name: networking.cilium.io usages: - client auth secretName: hubble-relay-client-certs {{ end }} {{ end }} --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: cilium-operator namespace: kube-system labels: io.cilium/app: operator name: cilium-operator spec: selector: matchLabels: io.cilium/app: operator name: cilium-operator maxUnavailable: 1