mirror of https://github.com/kubernetes/kops.git
1701 lines
48 KiB
Plaintext
1701 lines
48 KiB
Plaintext
{{ with .Networking.Cilium }}
|
|
{{ $semver := (trimPrefix "v" .Version) }}
|
|
{{ $healthPort := (ternary 9879 9876 (semverCompare ">=1.11.6" $semver)) }}
|
|
{{ $operatorHealthPort := 9234 }}
|
|
{{- if CiliumSecret }}
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: cilium-ipsec-keys
|
|
namespace: kube-system
|
|
stringData:
|
|
{{ CiliumSecret }}
|
|
---
|
|
{{- end }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: cilium
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: cilium-operator
|
|
namespace: kube-system
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: hubble-relay
|
|
namespace: kube-system
|
|
{{ end }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: cilium-config
|
|
namespace: kube-system
|
|
data:
|
|
agent-health-port: "{{ $healthPort }}"
|
|
|
|
{{- if .EtcdManaged }}
|
|
kvstore: etcd
|
|
kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}'
|
|
|
|
etcd-config: |-
|
|
---
|
|
endpoints:
|
|
- https://{{ APIInternalName }}:4003
|
|
|
|
trusted-ca-file: '/var/lib/etcd-secrets/etcd-ca.crt'
|
|
key-file: '/var/lib/etcd-secrets/etcd-client-cilium.key'
|
|
cert-file: '/var/lib/etcd-secrets/etcd-client-cilium.crt'
|
|
|
|
enable-k8s-event-handover: "true"
|
|
{{ end }}
|
|
|
|
# Identity allocation mode selects how identities are shared between cilium
|
|
# nodes by setting how they are stored. The options are "crd" or "kvstore".
|
|
# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
|
|
# These can be queried with:
|
|
# kubectl get ciliumid
|
|
# - "kvstore" stores identities in a kvstore, etcd or consul, that is
|
|
# configured below. Cilium versions before 1.6 supported only the kvstore
|
|
# backend. Upgrades from these older cilium versions should continue using
|
|
# the kvstore by commenting out the identity-allocation-mode below, or
|
|
# setting it to "kvstore".
|
|
# (default crd)
|
|
identity-allocation-mode: "{{ .IdentityAllocationMode }}"
|
|
|
|
# Time to wait before using new identity on endpoint identity change (default 5s)
|
|
identity-change-grace-period: "{{ .IdentityChangeGracePeriod }}"
|
|
|
|
# If you want to run cilium in debug mode change this value to true
|
|
debug: "{{ .Debug }}"
|
|
|
|
{{ if .EnablePrometheusMetrics }}
|
|
# If you want metrics enabled in all of your Cilium agents, set the port for
|
|
# which the Cilium agents will have their metrics exposed.
|
|
# This option deprecates the "prometheus-serve-addr" in the
|
|
# "cilium-metrics-config" ConfigMap
|
|
# NOTE that this will open the port on ALL nodes where Cilium pods are
|
|
# scheduled.
|
|
prometheus-serve-addr: ":{{ .AgentPrometheusPort }}"
|
|
operator-prometheus-serve-addr: ":6942"
|
|
enable-metrics: "true"
|
|
{{ end }}
|
|
|
|
{{ if .Metrics }}
|
|
# Metrics that should be enabled or disabled from the default metric
|
|
# list. (+metric_foo to enable metric_foo , -metric_bar to disable
|
|
# metric_bar).
|
|
metrics: {{- range .Metrics }}
|
|
{{ . }}
|
|
{{- end }}
|
|
{{ end }}
|
|
|
|
{{ if .EnableEncryption }}
|
|
{{ if eq .EncryptionType "ipsec" }}
|
|
enable-ipsec: "true"
|
|
ipsec-key-file: /etc/ipsec/keys
|
|
{{ else if eq .EncryptionType "wireguard" }}
|
|
enable-wireguard: "true"
|
|
{{ end }}
|
|
encrypt-node: "{{ .NodeEncryption }}"
|
|
{{ end }}
|
|
|
|
# Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
|
|
# address.
|
|
enable-ipv4: "{{ not IsIPv6Only }}"
|
|
# Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
|
|
# address.
|
|
enable-ipv6: "{{ IsIPv6Only }}"
|
|
# If you want cilium monitor to aggregate tracing for packets, set this level
|
|
# to "low", "medium", or "maximum". The higher the level, the less packets
|
|
# that will be seen in monitor output.
|
|
monitor-aggregation: "{{ .MonitorAggregation }}"
|
|
# ct-global-max-entries-* specifies the maximum number of connections
|
|
# supported across all endpoints, split by protocol: tcp or other. One pair
|
|
# of maps uses these values for IPv4 connections, and another pair of maps
|
|
# use these values for IPv6 connections.
|
|
#
|
|
# If these values are modified, then during the next Cilium startup the
|
|
# tracking of ongoing connections may be disrupted. This may lead to brief
|
|
# policy drops or a change in loadbalancing decisions for a connection.
|
|
#
|
|
# For users upgrading from Cilium 1.2 or earlier, to minimize disruption
|
|
# during the upgrade process, comment out these options.
|
|
bpf-ct-global-tcp-max: "{{ .BPFCTGlobalTCPMax }}"
|
|
bpf-ct-global-any-max: "{{ .BPFCTGlobalAnyMax }}"
|
|
|
|
# BPF load balancing algorithm ("random", "maglev") (default "random")
|
|
bpf-lb-algorithm: "{{ .BPFLBAlgorithm }}"
|
|
|
|
# Maglev per service backend table size (parameter M) (default 16381)
|
|
bpf-lb-maglev-table-size: "{{ .BPFLBMaglevTableSize }}"
|
|
|
|
# bpf-nat-global-max specified the maximum number of entries in the
|
|
# BPF NAT table. (default 524288)
|
|
bpf-nat-global-max: "{{ .BPFNATGlobalMax }}"
|
|
|
|
# bpf-neigh-global-max specified the maximum number of entries in the
|
|
# BPF neighbor table. (default 524288)
|
|
bpf-neigh-global-max: "{{ .BPFNeighGlobalMax }}"
|
|
|
|
# bpf-policy-map-max specifies the maximum number of entries in endpoint
|
|
# policy map (per endpoint) (default 16384)
|
|
bpf-policy-map-max: "{{ .BPFPolicyMapMax }}"
|
|
|
|
# bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
|
|
# backend and affinity maps. (default 65536)
|
|
bpf-lb-map-max: "{{ .BPFLBMapMax }}"
|
|
|
|
# bpf-lb-sock-hostns-only enables skipping socket LB for services when inside a pod namespace,
|
|
# in favor of service LB at the pod interface. Socket LB is still used when in the host namespace.
|
|
# Required by service mesh (e.g., Istio, Linkerd). (default false)
|
|
bpf-lb-sock-hostns-only: "{{ .BPFLBSockHostNSOnly }}"
|
|
|
|
{{ if .ChainingMode }}
|
|
cni-chaining-mode: "{{ .ChainingMode }}"
|
|
{{ end }}
|
|
|
|
# enable-bpf-masquerade enables masquerading packets from endpoints leaving
|
|
# the host with BPF instead of iptables. (default false)
|
|
enable-bpf-masquerade: "{{ and (WithDefaultBool .EnableBPFMasquerade false) (not IsIPv6Only) }}"
|
|
|
|
# Pre-allocation of map entries allows per-packet latency to be reduced, at
|
|
# the expense of up-front memory allocation for the entries in the maps. The
|
|
# default value below will minimize memory usage in the default installation;
|
|
# users who are sensitive to latency may consider setting this to "true".
|
|
#
|
|
# This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
|
|
# this option and behave as though it is set to "true".
|
|
#
|
|
# If this value is modified, then during the next Cilium startup the restore
|
|
# of existing endpoints and tracking of ongoing connections may be disrupted.
|
|
# This may lead to policy drops or a change in loadbalancing decisions for a
|
|
# connection for some time. Endpoints may need to be recreated to restore
|
|
# connectivity.
|
|
#
|
|
# If this option is set to "false" during an upgrade from 1.3 or earlier to
|
|
# 1.4 or later, then it may cause one-time disruptions during the upgrade.
|
|
preallocate-bpf-maps: "{{- if .PreallocateBPFMaps -}}true{{- else -}}false{{- end -}}"
|
|
# Regular expression matching compatible Istio sidecar istio-proxy
|
|
# container image names
|
|
sidecar-istio-proxy-image: "{{ .SidecarIstioProxyImage }}"
|
|
# Encapsulation mode for communication between nodes
|
|
# Possible values:
|
|
# - disabled
|
|
# - vxlan (default)
|
|
# - geneve
|
|
{{ if eq .Tunnel "disabled" }}
|
|
# This option enables native-routing mode, in place of tunnel=disabled, now deprecated.
|
|
routing-mode: "native"
|
|
{{ else }}
|
|
routing-mode: "tunnel"
|
|
tunnel-protocol: "{{ .Tunnel }}"
|
|
{{ end }}
|
|
|
|
# Name of the cluster. Only relevant when building a mesh of clusters.
|
|
cluster-name: "{{ .ClusterName }}"
|
|
# ID of the cluster. Only relevant when building a mesh of clusters.
|
|
# Must be a number between 1 and 255.
|
|
{{ if .ClusterID }}
|
|
cluster-id: "{{ .ClusterID }}"
|
|
{{ end }}
|
|
|
|
remove-cilium-node-taints: "true"
|
|
set-cilium-node-taints: "true"
|
|
set-cilium-is-up-condition: "true"
|
|
|
|
# DNS response code for rejecting DNS requests,
|
|
# available options are "nameError" and "refused"
|
|
tofqdns-dns-reject-response-code: "{{ .ToFQDNsDNSRejectResponseCode }}"
|
|
# This option is disabled by default starting from version 1.4.x in favor
|
|
# of a more powerful DNS proxy-based implementation, see [0] for details.
|
|
# Enable this option if you want to use FQDN policies but do not want to use
|
|
# the DNS proxy.
|
|
#
|
|
# To ease upgrade, users may opt to set this option to "true".
|
|
# Otherwise please refer to the Upgrade Guide [1] which explains how to
|
|
# prepare policy rules for upgrade.
|
|
#
|
|
# [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
|
|
# [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
|
|
tofqdns-enable-poller: "{{- if .ToFQDNsEnablePoller -}}true{{- else -}}false{{- end -}}"
|
|
{{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }}
|
|
# wait-bpf-mount makes init container wait until bpf filesystem is mounted
|
|
wait-bpf-mount: "false"
|
|
{{- end }}
|
|
# Enable fetching of container-runtime specific metadata
|
|
#
|
|
# By default, the Kubernetes pod and namespace labels are retrieved and
|
|
# associated with endpoints for identification purposes. By integrating
|
|
# with the container runtime, container runtime specific labels can be
|
|
# retrieved, such labels will be prefixed with container:
|
|
#
|
|
# CAUTION: The container runtime labels can include information such as pod
|
|
# annotations which may result in each pod being associated a unique set of
|
|
# labels which can result in excessive security identities being allocated.
|
|
# Please review the labels filter when enabling container runtime labels.
|
|
#
|
|
# Supported values:
|
|
# - containerd
|
|
# - crio
|
|
# - docker
|
|
# - none
|
|
# - auto (automatically detect the container runtime)
|
|
#
|
|
enable-ipv4-masquerade: "{{ .Masquerade }}"
|
|
enable-ipv6-masquerade: "false"
|
|
install-iptables-rules: "{{ WithDefaultBool .InstallIptablesRules true }}"
|
|
auto-direct-node-routes: "{{ .AutoDirectNodeRoutes }}"
|
|
{{ if .EnableHostReachableServices }}
|
|
enable-host-reachable-services: "{{ .EnableHostReachableServices }}"
|
|
{{ end }}
|
|
enable-node-port: "{{ .EnableNodePort }}"
|
|
kube-proxy-replacement: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
|
|
|
|
{{ with .IPAM }}
|
|
ipam: {{ . }}
|
|
{{ if eq . "eni" }}
|
|
enable-endpoint-routes: "true"
|
|
auto-create-cilium-node-resource: "true"
|
|
blacklist-conflicting-routes: "false"
|
|
{{ end }}
|
|
{{ end }}
|
|
|
|
# Disables usage of CiliumEndpoint CRD
|
|
disable-endpoint-crd: "{{ .DisableEndpointCRD }}"
|
|
|
|
# Enable connectivity health checking between virtual endpoints (default true)
|
|
enable-endpoint-health-checking: "{{ .EnableEndpointHealthChecking }}"
|
|
|
|
# Enable use of remote node identity (default false)
|
|
enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}"
|
|
|
|
# Enable unreachable routes on pod deletion (default false)
|
|
enable-unreachable-routes: "{{ .EnableUnreachableRoutes }}"
|
|
|
|
# enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true)
|
|
enable-l7-proxy: "{{ .EnableL7Proxy }}"
|
|
|
|
cgroup-root: /run/cilium/cgroupv2
|
|
|
|
disable-cnp-status-updates: "{{ .DisableCNPStatusUpdates }}"
|
|
nodes-gc-interval: "5m0s"
|
|
|
|
enable-service-topology: "{{ .EnableServiceTopology }}"
|
|
|
|
{{ if WithDefaultBool .Ingress.Enabled false }}
|
|
enable-envoy-config: "true"
|
|
external-envoy-proxy: "false"
|
|
enable-ingress-controller: "true"
|
|
ingress-secrets-namespace: kube-system
|
|
|
|
{{ if .Ingress.EnforceHttps }}
|
|
enforce-ingress-https: "{{ .Ingress.EnforceHttps }}"
|
|
{{ end }}
|
|
|
|
{{ if .Ingress.EnableSecretsSync }}
|
|
enable-ingress-secrets-sync: "{{ .Ingress.EnableSecretsSync }}"
|
|
{{ end }}
|
|
|
|
{{ if .Ingress.SharedLoadBalancerServiceName }}
|
|
ingress-shared-lb-service-name: {{ .Ingress.SharedLoadBalancerServiceName }}
|
|
{{ end }}
|
|
|
|
{{ if .Ingress.DefaultLoadBalancerMode }}
|
|
ingress-default-lb-mode: {{ .Ingress.DefaultLoadBalancerMode }}
|
|
{{ end }}
|
|
|
|
{{ if .Ingress.LoadBalancerAnnotationPrefixes }}
|
|
ingress-lb-annotation-prefixes: "{{ .Ingress.LoadBalancerAnnotationPrefixes }}"
|
|
{{ end }}
|
|
{{ end }}
|
|
|
|
# Tell the agent to generate and write a CNI configuration file
|
|
write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
|
|
cni-exclusive: "true"
|
|
cni-log-file: "/var/run/cilium/cilium-cni.log"
|
|
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
# Enable Hubble gRPC service.
|
|
enable-hubble: "true"
|
|
# UNIX domain socket for Hubble server to listen to.
|
|
hubble-socket-path: "/var/run/cilium/hubble.sock"
|
|
# An additional address for Hubble server to listen to (e.g. ":4244").
|
|
hubble-listen-address: ":4244"
|
|
hubble-disable-tls: "false"
|
|
hubble-tls-cert-file: /var/lib/cilium/tls/hubble/tls.crt
|
|
hubble-tls-key-file: /var/lib/cilium/tls/hubble/tls.key
|
|
hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/ca.crt
|
|
{{ if .Hubble.Metrics }}
|
|
hubble-metrics-server: ":9091"
|
|
hubble-metrics:
|
|
{{- range .Hubble.Metrics }}
|
|
{{ . }}
|
|
{{- end }}
|
|
{{ end }}
|
|
{{ end }}
|
|
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
---
|
|
# Source: cilium/templates/hubble-relay-configmap.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: hubble-relay-config
|
|
namespace: kube-system
|
|
data:
|
|
config.yaml: |
|
|
peer-service: unix:///var/run/cilium/hubble.sock
|
|
listen-address: :4245
|
|
|
|
disable-server-tls: true
|
|
|
|
tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
|
|
tls-client-key-file: /var/lib/hubble-relay/tls/client.key
|
|
tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt
|
|
|
|
{{ end }}
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: cilium
|
|
rules:
|
|
- apiGroups:
|
|
- networking.k8s.io
|
|
resources:
|
|
- networkpolicies
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- discovery.k8s.io
|
|
resources:
|
|
- endpointslices
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- namespaces
|
|
- services
|
|
- pods
|
|
- endpoints
|
|
- nodes
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- apiextensions.k8s.io
|
|
resources:
|
|
- customresourcedefinitions
|
|
verbs:
|
|
- list
|
|
- watch
|
|
- get
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumloadbalancerippools
|
|
- ciliumbgppeeringpolicies
|
|
- ciliumclusterwideenvoyconfigs
|
|
- ciliumclusterwidenetworkpolicies
|
|
- ciliumegressgatewaypolicies
|
|
- ciliumendpoints
|
|
- ciliumendpointslices
|
|
- ciliumenvoyconfigs
|
|
- ciliumidentities
|
|
- ciliumlocalredirectpolicies
|
|
- ciliumnetworkpolicies
|
|
- ciliumnodes
|
|
- ciliumnodeconfigs
|
|
- ciliumcidrgroups
|
|
- ciliuml2announcementpolicies
|
|
- ciliumpodippools
|
|
verbs:
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumidentities
|
|
- ciliumendpoints
|
|
- ciliumnodes
|
|
verbs:
|
|
- create
|
|
- apiGroups:
|
|
- cilium.io
|
|
# To synchronize garbage collection of such resources
|
|
resources:
|
|
- ciliumidentities
|
|
verbs:
|
|
- update
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumendpoints
|
|
verbs:
|
|
- delete
|
|
- get
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnodes
|
|
- ciliumnodes/status
|
|
verbs:
|
|
- get
|
|
- update
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnetworkpolicies/status
|
|
- ciliumclusterwidenetworkpolicies/status
|
|
- ciliumendpoints/status
|
|
- ciliumendpoints
|
|
- ciliuml2announcementpolicies/status
|
|
verbs:
|
|
- patch
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: cilium-operator
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- pods
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
# to automatically delete [core|kube]dns pods so that are starting to being
|
|
# managed by Cilium
|
|
- delete
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- nodes
|
|
verbs:
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
# To remove node taints
|
|
- nodes
|
|
# To set NetworkUnavailable false on startup
|
|
- nodes/status
|
|
verbs:
|
|
- patch
|
|
- apiGroups:
|
|
- discovery.k8s.io
|
|
resources:
|
|
- endpointslices
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
# to perform LB IP allocation for BGP
|
|
- services/status
|
|
verbs:
|
|
- update
|
|
- patch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
# to check apiserver connectivity
|
|
- namespaces
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
# to perform the translation of a CNP that contains `ToGroup` to its endpoints
|
|
- services
|
|
- endpoints
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnetworkpolicies
|
|
- ciliumclusterwidenetworkpolicies
|
|
verbs:
|
|
# Create auto-generated CNPs and CCNPs from Policies that have 'toGroups'
|
|
- create
|
|
- update
|
|
- deletecollection
|
|
# To update the status of the CNPs and CCNPs
|
|
- patch
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnetworkpolicies/status
|
|
- ciliumclusterwidenetworkpolicies/status
|
|
verbs:
|
|
# Update the auto-generated CNPs and CCNPs status.
|
|
- patch
|
|
- update
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumendpoints
|
|
- ciliumidentities
|
|
verbs:
|
|
# To perform garbage collection of such resources
|
|
- delete
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumidentities
|
|
verbs:
|
|
# To synchronize garbage collection of such resources
|
|
- update
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnodes
|
|
verbs:
|
|
- create
|
|
- update
|
|
- get
|
|
- list
|
|
- watch
|
|
# To perform CiliumNode garbage collector
|
|
- delete
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumnodes/status
|
|
verbs:
|
|
- update
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumendpointslices
|
|
- ciliumenvoyconfigs
|
|
verbs:
|
|
- create
|
|
- update
|
|
- get
|
|
- list
|
|
- watch
|
|
- delete
|
|
- patch
|
|
- apiGroups:
|
|
- apiextensions.k8s.io
|
|
resources:
|
|
- customresourcedefinitions
|
|
verbs:
|
|
- create
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- apiextensions.k8s.io
|
|
resources:
|
|
- customresourcedefinitions
|
|
verbs:
|
|
- update
|
|
resourceNames:
|
|
- ciliumloadbalancerippools.cilium.io
|
|
- ciliumbgppeeringpolicies.cilium.io
|
|
- ciliumclusterwideenvoyconfigs.cilium.io
|
|
- ciliumclusterwidenetworkpolicies.cilium.io
|
|
- ciliumegressgatewaypolicies.cilium.io
|
|
- ciliumendpoints.cilium.io
|
|
- ciliumendpointslices.cilium.io
|
|
- ciliumenvoyconfigs.cilium.io
|
|
- ciliumexternalworkloads.cilium.io
|
|
- ciliumidentities.cilium.io
|
|
- ciliumlocalredirectpolicies.cilium.io
|
|
- ciliumnetworkpolicies.cilium.io
|
|
- ciliumnodes.cilium.io
|
|
- ciliumnodeconfigs.cilium.io
|
|
- ciliumcidrgroups.cilium.io
|
|
- ciliuml2announcementpolicies.cilium.io
|
|
- ciliumpodippools.cilium.io
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumloadbalancerippools
|
|
- ciliumpodippools
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumpodippools
|
|
verbs:
|
|
- create
|
|
- apiGroups:
|
|
- cilium.io
|
|
resources:
|
|
- ciliumloadbalancerippools/status
|
|
verbs:
|
|
- patch
|
|
- apiGroups:
|
|
- coordination.k8s.io
|
|
resources:
|
|
- leases
|
|
verbs:
|
|
- create
|
|
- get
|
|
- update
|
|
{{ if WithDefaultBool .Ingress.Enabled false }}
|
|
- apiGroups:
|
|
- networking.k8s.io
|
|
resources:
|
|
- ingresses
|
|
- ingressclasses
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- networking.k8s.io
|
|
resources:
|
|
- ingresses/status # To update ingress status with load balancer IP.
|
|
verbs:
|
|
- update
|
|
{{ end }}
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: cilium
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: cilium
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: cilium
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: cilium-operator
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: cilium-operator
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: cilium-operator
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: cilium-config-agent
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- configmaps
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
---
|
|
# Source: cilium/templates/cilium-agent/rolebinding.yaml
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: cilium-config-agent
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: cilium-config-agent
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: "cilium"
|
|
namespace: kube-system
|
|
{{ if WithDefaultBool .Ingress.Enabled false }}
|
|
---
|
|
# Source: cilium/templates/cilium-agent/role.yaml
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: cilium-ingress-secrets
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: cilium-secrets
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: cilium-ingress-secrets
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: "cilium"
|
|
namespace: kube-system
|
|
---
|
|
# Source: cilium/templates/cilium-operator/role.yaml
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: cilium-operator-ingress-secrets
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
rules:
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
verbs:
|
|
- create
|
|
- delete
|
|
- update
|
|
- patch
|
|
---
|
|
# Source: cilium/templates/cilium-operator/rolebinding.yaml
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: cilium-operator-ingress-secrets
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/part-of: cilium
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: cilium-operator-ingress-secrets
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: "cilium-operator"
|
|
namespace: kube-system
|
|
---
|
|
# Source: cilium/templates/cilium-ingress-class.yaml
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: IngressClass
|
|
metadata:
|
|
name: cilium
|
|
spec:
|
|
controller: cilium.io/ingress-controller
|
|
{{ if or (eq .Ingress.DefaultLoadBalancerMode "shared") (not .Ingress.DefaultLoadBalancerMode) }}
|
|
---
|
|
# Source: cilium/templates/cilium-ingress-service.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: {{ .Ingress.SharedLoadBalancerServiceName }}
|
|
namespace: kube-system
|
|
labels:
|
|
cilium.io/ingress: "true"
|
|
spec:
|
|
ports:
|
|
- name: http
|
|
port: 80
|
|
protocol: TCP
|
|
nodePort:
|
|
- name: https
|
|
port: 443
|
|
protocol: TCP
|
|
nodePort:
|
|
type: LoadBalancer
|
|
---
|
|
# Source: cilium/templates/cilium-ingress-service.yaml
|
|
apiVersion: v1
|
|
kind: Endpoints
|
|
metadata:
|
|
name: {{ .Ingress.SharedLoadBalancerServiceName }}
|
|
namespace: kube-system
|
|
subsets:
|
|
- addresses:
|
|
- ip: "192.192.192.192"
|
|
ports:
|
|
- port: 9999
|
|
{{ end }}
|
|
{{ end }}
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
---
|
|
# Source: cilium/templates/hubble-relay-service.yaml
|
|
kind: Service
|
|
apiVersion: v1
|
|
metadata:
|
|
name: hubble-relay
|
|
namespace: kube-system
|
|
labels:
|
|
k8s-app: hubble-relay
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
k8s-app: hubble-relay
|
|
ports:
|
|
- protocol: TCP
|
|
port: 80
|
|
targetPort: 4245
|
|
{{ end }}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: cilium
|
|
namespace: kube-system
|
|
labels:
|
|
k8s-app: cilium
|
|
kubernetes.io/cluster-service: "true"
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
k8s-app: cilium
|
|
kubernetes.io/cluster-service: "true"
|
|
updateStrategy:
|
|
rollingUpdate:
|
|
maxUnavailable: 2
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
{{ if .EnablePrometheusMetrics }}
|
|
# Annotation required for prometheus auto-discovery scraping
|
|
# https://docs.cilium.io/en/v1.9/operations/metrics/#installation
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "{{ .AgentPrometheusPort }}"
|
|
{{ end }}
|
|
{{- range $key, $value := .AgentPodAnnotations }}
|
|
{{ $key }}: "{{ $value }}"
|
|
{{- end }}
|
|
labels:
|
|
k8s-app: cilium
|
|
kubernetes.io/cluster-service: "true"
|
|
spec:
|
|
containers:
|
|
- name: cilium-agent
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- cilium-agent
|
|
args:
|
|
- --config-dir=/tmp/cilium/config-map
|
|
startupProbe:
|
|
httpGet:
|
|
host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}'
|
|
path: /healthz
|
|
port: {{ $healthPort }}
|
|
scheme: HTTP
|
|
httpHeaders:
|
|
- name: "brief"
|
|
value: "true"
|
|
failureThreshold: 105
|
|
periodSeconds: 2
|
|
successThreshold: 1
|
|
livenessProbe:
|
|
httpGet:
|
|
host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}'
|
|
path: /healthz
|
|
port: {{ $healthPort }}
|
|
scheme: HTTP
|
|
httpHeaders:
|
|
- name: "brief"
|
|
value: "true"
|
|
periodSeconds: 30
|
|
successThreshold: 1
|
|
failureThreshold: 10
|
|
timeoutSeconds: 5
|
|
resources:
|
|
requests:
|
|
cpu: {{ or .CPURequest "25m" }}
|
|
memory: {{ or .MemoryRequest "128Mi" }}
|
|
readinessProbe:
|
|
httpGet:
|
|
host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}'
|
|
path: /healthz
|
|
port: {{ $healthPort }}
|
|
scheme: HTTP
|
|
httpHeaders:
|
|
- name: "brief"
|
|
value: "true"
|
|
periodSeconds: 30
|
|
successThreshold: 1
|
|
failureThreshold: 3
|
|
timeoutSeconds: 5
|
|
env:
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: spec.nodeName
|
|
- name: CILIUM_K8S_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: metadata.namespace
|
|
- name: CILIUM_CLUSTERMESH_CONFIG
|
|
value: /var/lib/cilium/clustermesh/
|
|
- name: CILIUM_CNI_CHAINING_MODE
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
key: cni-chaining-mode
|
|
name: cilium-config
|
|
optional: true
|
|
- name: CILIUM_CUSTOM_CNI_CONF
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
key: custom-cni-conf
|
|
name: cilium-config
|
|
optional: true
|
|
- name: KUBERNETES_SERVICE_HOST
|
|
value: "{{ APIInternalName }}"
|
|
- name: KUBERNETES_SERVICE_PORT
|
|
value: "443"
|
|
{{ with .EnablePolicy }}
|
|
- name: CILIUM_ENABLE_POLICY
|
|
value: {{ . }}
|
|
{{ end }}
|
|
lifecycle:
|
|
{{ if eq .IPAM "eni" }}
|
|
postStart:
|
|
exec:
|
|
command:
|
|
- "bash"
|
|
- "-c"
|
|
- |
|
|
set -o errexit
|
|
set -o pipefail
|
|
set -o nounset
|
|
|
|
# When running in AWS ENI mode, it's likely that 'aws-node' has
|
|
# had a chance to install SNAT iptables rules. These can result
|
|
# in dropped traffic, so we should attempt to remove them.
|
|
# We do it using a 'postStart' hook since this may need to run
|
|
# for nodes which might have already been init'ed but may still
|
|
# have dangling rules. This is safe because there are no
|
|
# dependencies on anything that is part of the startup script
|
|
# itself, and can be safely run multiple times per node (e.g. in
|
|
# case of a restart).
|
|
if [[ "$(iptables-save | grep -c AWS-SNAT-CHAIN)" != "0" ]];
|
|
then
|
|
echo 'Deleting iptables rules created by the AWS CNI VPC plugin'
|
|
iptables-save | grep -v AWS-SNAT-CHAIN | iptables-restore
|
|
fi
|
|
echo 'Done!'
|
|
{{- end }}
|
|
preStop:
|
|
exec:
|
|
command:
|
|
- /cni-uninstall.sh
|
|
ports:
|
|
{{ if .EnablePrometheusMetrics }}
|
|
- containerPort: {{ .AgentPrometheusPort }}
|
|
name: prometheus
|
|
protocol: TCP
|
|
{{ end }}
|
|
{{- if .Hubble.Metrics }}
|
|
- containerPort: 9091
|
|
hostPort: 9091
|
|
name: hubble-metrics
|
|
protocol: TCP
|
|
{{- end }}
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- name: bpf-maps
|
|
mountPath: /sys/fs/bpf
|
|
{{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }}
|
|
mountPropagation: Bidirectional
|
|
{{- end }}
|
|
- name: cilium-cgroup
|
|
mountPath: /run/cilium/cgroupv2
|
|
- name: cilium-run
|
|
mountPath: /var/run/cilium
|
|
{{- if not (semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver) }}
|
|
- name: cni-path
|
|
mountPath: /host/opt/cni/bin
|
|
{{- end }}
|
|
- name: etc-cni-netd
|
|
mountPath: /host/etc/cni/net.d
|
|
{{ if .EtcdManaged }}
|
|
- name: etcd-config-path
|
|
mountPath: /var/lib/etcd-config
|
|
readOnly: true
|
|
- name: etcd-secrets
|
|
mountPath: /var/lib/etcd-secrets
|
|
readOnly: true
|
|
{{ end }}
|
|
- name: clustermesh-secrets
|
|
mountPath: /var/lib/cilium/clustermesh
|
|
readOnly: true
|
|
- name: cilium-config-path
|
|
mountPath: /tmp/cilium/config-map
|
|
readOnly: true
|
|
# Needed to be able to load kernel modules
|
|
- name: lib-modules
|
|
mountPath: /lib/modules
|
|
readOnly: true
|
|
- name: xtables-lock
|
|
mountPath: /run/xtables.lock
|
|
- name: tmp
|
|
mountPath: /tmp
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
- name: hubble-tls
|
|
mountPath: /var/lib/cilium/tls/hubble
|
|
readOnly: true
|
|
{{ end }}
|
|
{{ if CiliumSecret }}
|
|
- mountPath: /etc/ipsec
|
|
name: cilium-ipsec-secrets
|
|
{{ end }}
|
|
{{ if .Debug }}
|
|
- name: cilium-monitor
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /bin/bash
|
|
- -c
|
|
- --
|
|
args:
|
|
- |-
|
|
for i in {1..5}; do \
|
|
[ -S /var/run/cilium/monitor1_2.sock ] && break || sleep 10;\
|
|
done; \
|
|
cilium monitor --type=agent
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- name: cilium-run
|
|
mountPath: /var/run/cilium
|
|
{{ end }}
|
|
initContainers:
|
|
- name: config
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- cilium
|
|
- build-config
|
|
env:
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: spec.nodeName
|
|
- name: CILIUM_K8S_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: metadata.namespace
|
|
- name: KUBERNETES_SERVICE_HOST
|
|
value: "{{ APIInternalName }}"
|
|
- name: KUBERNETES_SERVICE_PORT
|
|
value: "443"
|
|
volumeMounts:
|
|
- name: tmp
|
|
mountPath: /tmp
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
# Required to mount cgroup2 filesystem on the underlying Kubernetes node.
|
|
# We use nsenter command with host's cgroup and mount namespaces enabled.
|
|
- name: mount-cgroup
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
- name: CGROUP_ROOT
|
|
value: /run/cilium/cgroupv2
|
|
- name: BIN_PATH
|
|
value: /opt/cni/bin
|
|
command:
|
|
- sh
|
|
- -ec
|
|
# The statically linked Go program binary is invoked to avoid any
|
|
# dependency on utilities like sh and mount that can be missing on certain
|
|
# distros installed on the underlying host. Copy the binary to the
|
|
# same directory where we install cilium cni plugin so that exec permissions
|
|
# are available.
|
|
- |
|
|
cp /usr/bin/cilium-mount /hostbin/cilium-mount;
|
|
nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT;
|
|
rm /hostbin/cilium-mount
|
|
volumeMounts:
|
|
- name: hostproc
|
|
mountPath: /hostproc
|
|
- name: cni-path
|
|
mountPath: /hostbin
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
privileged: true
|
|
- name: apply-sysctl-overwrites
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
- name: BIN_PATH
|
|
value: /opt/cni/bin
|
|
command:
|
|
- sh
|
|
- -ec
|
|
# The statically linked Go program binary is invoked to avoid any
|
|
# dependency on utilities like sh that can be missing on certain
|
|
# distros installed on the underlying host. Copy the binary to the
|
|
# same directory where we install cilium cni plugin so that exec permissions
|
|
# are available.
|
|
- |
|
|
cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
|
|
nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
|
|
rm /hostbin/cilium-sysctlfix
|
|
volumeMounts:
|
|
- name: hostproc
|
|
mountPath: /hostproc
|
|
- name: cni-path
|
|
mountPath: /hostbin
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
privileged: true
|
|
- name: clean-cilium-state
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /init-container.sh
|
|
env:
|
|
- name: CILIUM_ALL_STATE
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: cilium-config
|
|
key: clean-cilium-state
|
|
optional: true
|
|
- name: CILIUM_BPF_STATE
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: cilium-config
|
|
key: clean-cilium-bpf-state
|
|
optional: true
|
|
- name: KUBERNETES_SERVICE_HOST
|
|
value: "{{ APIInternalName }}"
|
|
- name: KUBERNETES_SERVICE_PORT
|
|
value: "443"
|
|
{{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }}
|
|
- name: CILIUM_WAIT_BPF_MOUNT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
key: wait-bpf-mount
|
|
name: cilium-config
|
|
optional: true
|
|
{{- end }}
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
privileged: true
|
|
volumeMounts:
|
|
- name: bpf-maps
|
|
mountPath: /sys/fs/bpf
|
|
{{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }}
|
|
mountPropagation: HostToContainer
|
|
{{- end }}
|
|
# Required to mount cgroup filesystem from the host to cilium agent pod
|
|
- name: cilium-cgroup
|
|
mountPath: /run/cilium/cgroupv2
|
|
mountPropagation: HostToContainer
|
|
- name: cilium-run
|
|
mountPath: /var/run/cilium
|
|
{{- if semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver }}
|
|
# Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent
|
|
- name: install-cni-binaries
|
|
image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /install-plugin.sh
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 10Mi
|
|
securityContext:
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
terminationMessagePath: /dev/termination-log
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- name: cni-path
|
|
mountPath: /host/opt/cni/bin
|
|
{{- end }}
|
|
restartPolicy: Always
|
|
priorityClassName: system-node-critical
|
|
{{ if ContainerdSELinuxEnabled }}
|
|
securityContext:
|
|
seLinuxOptions:
|
|
type: spc_t
|
|
level: s0
|
|
{{ end }}
|
|
serviceAccount: cilium
|
|
serviceAccountName: cilium
|
|
terminationGracePeriodSeconds: 1
|
|
hostNetwork: true
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
k8s-app: cilium
|
|
topologyKey: kubernetes.io/hostname
|
|
nodeSelector:
|
|
kubernetes.io/os: linux
|
|
tolerations:
|
|
- operator: Exists
|
|
volumes:
|
|
# For sharing configuration between the "config" initContainer and the agent
|
|
- name: tmp
|
|
emptyDir: {}
|
|
# To keep state between restarts / upgrades
|
|
- name: cilium-run
|
|
hostPath:
|
|
path: /var/run/cilium
|
|
type: DirectoryOrCreate
|
|
# To keep state between restarts / upgrades for bpf maps
|
|
- name: bpf-maps
|
|
hostPath:
|
|
path: /sys/fs/bpf
|
|
type: DirectoryOrCreate
|
|
# To mount cgroup2 filesystem on the host
|
|
- name: hostproc
|
|
hostPath:
|
|
path: /proc
|
|
type: Directory
|
|
# To keep state between restarts / upgrades for cgroup2 filesystem
|
|
- name: cilium-cgroup
|
|
hostPath:
|
|
path: /run/cilium/cgroupv2
|
|
type: DirectoryOrCreate
|
|
# To install cilium cni plugin in the host
|
|
- name: cni-path
|
|
hostPath:
|
|
path: /opt/cni/bin
|
|
type: DirectoryOrCreate
|
|
# To install cilium cni configuration in the host
|
|
- name: etc-cni-netd
|
|
hostPath:
|
|
path: /etc/cni/net.d
|
|
type: DirectoryOrCreate
|
|
# To be able to load kernel modules
|
|
- name: lib-modules
|
|
hostPath:
|
|
path: /lib/modules
|
|
# To access iptables concurrently with other processes (e.g. kube-proxy)
|
|
- name: xtables-lock
|
|
hostPath:
|
|
path: /run/xtables.lock
|
|
type: FileOrCreate
|
|
{{- if .EtcdManaged }}
|
|
# To read the etcd config stored in config maps
|
|
- name: etcd-config-path
|
|
configMap:
|
|
name: cilium-config
|
|
# note: the leading zero means this number is in octal representation: do not remove it
|
|
defaultMode: 0400
|
|
items:
|
|
- key: etcd-config
|
|
path: etcd.config
|
|
# To read the Cilium etcd secrets in case the user might want to use TLS
|
|
- name: etcd-secrets
|
|
hostPath:
|
|
path: /etc/kubernetes/pki/cilium
|
|
type: Directory
|
|
{{- end }}
|
|
- name: clustermesh-secrets
|
|
projected:
|
|
# note: the leading zero means this number is in octal representation: do not remove it
|
|
defaultMode: 0400
|
|
sources:
|
|
- secret:
|
|
name: cilium-clustermesh
|
|
optional: true
|
|
# note: items are not explicitly listed here, since the entries of this secret
|
|
# depend on the peers configured, and that would cause a restart of all agents
|
|
# at every addition/removal. Leaving the field empty makes each secret entry
|
|
# to be automatically projected into the volume as a file whose name is the key.
|
|
- secret:
|
|
name: clustermesh-apiserver-remote-cert
|
|
optional: true
|
|
items:
|
|
- key: tls.key
|
|
path: common-etcd-client.key
|
|
- key: tls.crt
|
|
path: common-etcd-client.crt
|
|
- key: ca.crt
|
|
path: common-etcd-client-ca.crt
|
|
- configMap:
|
|
name: cilium-config
|
|
name: cilium-config-path
|
|
{{ if CiliumSecret }}
|
|
- name: cilium-ipsec-secrets
|
|
secret:
|
|
secretName: cilium-ipsec-keys
|
|
{{ end }}
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
- name: hubble-tls
|
|
projected:
|
|
# note: the leading zero means this number is in octal representation: do not remove it
|
|
defaultMode: 0400
|
|
sources:
|
|
- secret:
|
|
name: hubble-server-certs
|
|
optional: true
|
|
{{ end }}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
labels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
name: cilium-operator
|
|
namespace: kube-system
|
|
spec:
|
|
replicas: {{ ControlPlaneControllerReplicas false }}
|
|
selector:
|
|
matchLabels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
strategy:
|
|
rollingUpdate:
|
|
maxSurge: 1
|
|
maxUnavailable: 1
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
{{- range $key, $value := .OperatorPodAnnotations }}
|
|
{{ $key }}: "{{ $value }}"
|
|
{{- end }}
|
|
labels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
spec:
|
|
nodeSelector: null
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
- matchExpressions:
|
|
- key: node-role.kubernetes.io/master
|
|
operator: Exists
|
|
containers:
|
|
- name: cilium-operator
|
|
image: "{{ or .Registry "quay.io" }}/cilium/operator:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- cilium-operator
|
|
args:
|
|
- "--config-dir=/tmp/cilium/config-map"
|
|
- "--debug=$(CILIUM_DEBUG)"
|
|
- "--eni-tags={{ CloudLabels }}"
|
|
env:
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: spec.nodeName
|
|
- name: CILIUM_K8S_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: metadata.namespace
|
|
- name: CILIUM_DEBUG
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
key: debug
|
|
name: cilium-config
|
|
optional: true
|
|
- name: KUBERNETES_SERVICE_HOST
|
|
value: "{{ APIInternalName }}"
|
|
- name: KUBERNETES_SERVICE_PORT
|
|
value: "443"
|
|
{{ if .EnablePrometheusMetrics }}
|
|
ports:
|
|
- containerPort: 6942
|
|
hostPort: 6942
|
|
name: prometheus
|
|
protocol: TCP
|
|
{{ end }}
|
|
resources:
|
|
requests:
|
|
cpu: {{ or .CPURequest "25m" }}
|
|
memory: {{ or .MemoryRequest "128Mi" }}
|
|
livenessProbe:
|
|
httpGet:
|
|
host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}'
|
|
path: /healthz
|
|
port: {{ $operatorHealthPort }}
|
|
scheme: HTTP
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 10
|
|
timeoutSeconds: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}'
|
|
path: /healthz
|
|
port: {{ $operatorHealthPort }}
|
|
scheme: HTTP
|
|
initialDelaySeconds: 0
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 5
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- mountPath: /tmp/cilium/config-map
|
|
name: cilium-config-path
|
|
readOnly: true
|
|
{{- if .EtcdManaged }}
|
|
- mountPath: /var/lib/etcd-config
|
|
name: etcd-config-path
|
|
readOnly: true
|
|
- mountPath: /var/lib/etcd-secrets
|
|
name: etcd-secrets
|
|
readOnly: true
|
|
{{- end }}
|
|
hostNetwork: true
|
|
restartPolicy: Always
|
|
priorityClassName: system-cluster-critical
|
|
{{ if ContainerdSELinuxEnabled }}
|
|
securityContext:
|
|
seLinuxOptions:
|
|
type: spc_t
|
|
level: s0
|
|
{{ end }}
|
|
serviceAccount: cilium-operator
|
|
serviceAccountName: cilium-operator
|
|
tolerations:
|
|
- operator: Exists
|
|
topologySpreadConstraints:
|
|
- maxSkew: 1
|
|
topologyKey: "topology.kubernetes.io/zone"
|
|
whenUnsatisfiable: ScheduleAnyway
|
|
labelSelector:
|
|
matchLabels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
- maxSkew: 1
|
|
topologyKey: "kubernetes.io/hostname"
|
|
whenUnsatisfiable: DoNotSchedule
|
|
labelSelector:
|
|
matchLabels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
volumes:
|
|
# To read the configuration from the config map
|
|
- configMap:
|
|
name: cilium-config
|
|
name: cilium-config-path
|
|
{{- if .EtcdManaged }}
|
|
# To read the etcd config stored in config maps
|
|
- configMap:
|
|
defaultMode: 420
|
|
items:
|
|
- key: etcd-config
|
|
path: etcd.config
|
|
name: cilium-config
|
|
name: etcd-config-path
|
|
# To read the k8s etcd secrets in case the user might want to use TLS
|
|
- name: etcd-secrets
|
|
hostPath:
|
|
path: /etc/kubernetes/pki/cilium
|
|
type: Directory
|
|
{{- end }}
|
|
{{ if WithDefaultBool .Hubble.Enabled false }}
|
|
---
|
|
# Source: cilium/charts/hubble-relay/templates/deployment.yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: hubble-relay
|
|
namespace: kube-system
|
|
labels:
|
|
k8s-app: hubble-relay
|
|
spec:
|
|
replicas: 2
|
|
selector:
|
|
matchLabels:
|
|
k8s-app: hubble-relay
|
|
strategy:
|
|
rollingUpdate:
|
|
maxUnavailable: 1
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
k8s-app: hubble-relay
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 65532
|
|
containers:
|
|
- name: hubble-relay
|
|
image: "{{ or .Registry "quay.io" }}/cilium/hubble-relay:{{ .Version }}"
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
runAsGroup: 65532
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
command:
|
|
- hubble-relay
|
|
args:
|
|
- "serve"
|
|
- "--peer-service=unix:///var/run/cilium/hubble.sock"
|
|
- "--listen-address=:4245"
|
|
env:
|
|
# unfortunately, the addon CAs use only CN
|
|
- name: GODEBUG
|
|
value: x509ignoreCN=0
|
|
ports:
|
|
- name: grpc
|
|
containerPort: 4245
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: grpc
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: grpc
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- mountPath: /var/run/cilium
|
|
name: hubble-sock-dir
|
|
readOnly: true
|
|
- mountPath: /etc/hubble-relay
|
|
name: config
|
|
readOnly: true
|
|
- mountPath: /var/lib/hubble-relay/tls
|
|
name: tls
|
|
readOnly: true
|
|
restartPolicy: Always
|
|
serviceAccount: hubble-relay
|
|
serviceAccountName: hubble-relay
|
|
terminationGracePeriodSeconds: 0
|
|
topologySpreadConstraints:
|
|
- maxSkew: 1
|
|
topologyKey: "topology.kubernetes.io/zone"
|
|
whenUnsatisfiable: ScheduleAnyway
|
|
labelSelector:
|
|
matchLabels:
|
|
k8s-app: hubble-relay
|
|
- maxSkew: 1
|
|
topologyKey: "kubernetes.io/hostname"
|
|
whenUnsatisfiable: DoNotSchedule
|
|
labelSelector:
|
|
matchLabels:
|
|
k8s-app: hubble-relay
|
|
volumes:
|
|
- hostPath:
|
|
path: /var/run/cilium
|
|
type: Directory
|
|
name: hubble-sock-dir
|
|
- configMap:
|
|
name: hubble-relay-config
|
|
items:
|
|
- key: config.yaml
|
|
path: config.yaml
|
|
name: config
|
|
- projected:
|
|
sources:
|
|
- secret:
|
|
name: hubble-relay-client-certs
|
|
items:
|
|
- key: tls.crt
|
|
path: client.crt
|
|
- key: tls.key
|
|
path: client.key
|
|
- key: ca.crt
|
|
path: hubble-server-ca.crt
|
|
name: tls
|
|
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
labels:
|
|
k8s-app: cilium
|
|
name: hubble-server-certs
|
|
namespace: kube-system
|
|
spec:
|
|
dnsNames:
|
|
- "*.{{ replace .ClusterName "." "-" }}.hubble-grpc.cilium.io"
|
|
issuerRef:
|
|
kind: Issuer
|
|
name: networking.cilium.io
|
|
secretName: hubble-server-certs
|
|
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
labels:
|
|
k8s-app: cilium
|
|
name: hubble-relay-client-certs
|
|
namespace: kube-system
|
|
spec:
|
|
dnsNames:
|
|
- "hubble-relay-client"
|
|
issuerRef:
|
|
kind: Issuer
|
|
name: networking.cilium.io
|
|
usages:
|
|
- client auth
|
|
secretName: hubble-relay-client-certs
|
|
{{ end }}
|
|
{{ end }}
|
|
---
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata:
|
|
name: cilium-operator
|
|
namespace: kube-system
|
|
labels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
io.cilium/app: operator
|
|
name: cilium-operator
|
|
maxUnavailable: 1
|