diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index d98c9031e7..2976d0b18a 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -3427,6 +3427,31 @@ spec: description: 'BPFCTGlobalTCPMax is the maximum number of entries in the TCP CT table. Default: 524288' type: integer + bpfLBAlgorithm: + description: 'BPFLBAlgorithm is the load balancing algorithm + ("random", "maglev"). Default: random' + type: string + bpfLBMaglevTableSize: + description: 'BPFLBMaglevTableSize is the per service backend + table size when going with Maglev (parameter M). Default: + 16381' + type: string + bpfLBMapMax: + description: 'BPFLBMapMax is the maximum number of entries + in bpf lb service, backend and affinity maps. Default: 65536' + type: integer + bpfNATGlobalMax: + description: 'BPFNATGlobalMax is the the maximum number of + entries in the BPF NAT table. Default: 524288' + type: integer + bpfNeighGlobalMax: + description: 'BPFNeighGlobalMax is the the maximum number + of entries in the BPF Neighbor table. Default: 524288' + type: integer + bpfPolicyMapMax: + description: 'BPFPolicyMapMax is the maximum number of entries + in endpoint policy map. Default: 16384' + type: integer bpfRoot: description: BPFRoot is not implemented and may be removed in the future. Setting this has no effect. @@ -3484,6 +3509,10 @@ spec: description: DisableConntrack is not implemented and may be removed in the future. Setting this has no effect. type: boolean + disableEndpointCRD: + description: 'DisableEndpointCRD disables usage of CiliumEndpoint + CRD. Default: false' + type: boolean disableIpv4: description: 'DisableIpv4 is deprecated: Use EnableIpv4 instead. Setting this flag has no effect.' @@ -3496,16 +3525,29 @@ spec: description: DisableMasquerade disables masquerading traffic to external destinations behind the node IP. type: boolean + enableBPFMasquerade: + description: 'EnableBPFMasquerade enables masquerading packets + from endpoints leaving the host with BPF instead of iptables. + Default: false' + type: boolean enableEncryption: description: 'EnableEncryption enables Cilium Encryption. Default: false' type: boolean + enableEndpointHealthChecking: + description: 'EnableEndpointHealthChecking enables connectivity + health checking between virtual endpoints. Default: true' + type: boolean enableHostReachableServices: description: 'EnableHostReachableServices configures Cilium to enable services to be reached from the host namespace in addition to pod namespaces. https://docs.cilium.io/en/v1.9/gettingstarted/host-services/ Default: false' type: boolean + enableL7Proxy: + description: 'EnableL7Proxy enables L7 proxy for L7 policy + enforcement. Default: true' + type: boolean enableNodePort: description: 'EnableNodePort replaces kube-proxy with Cilium''s BPF implementation. Requires spec.kubeProxy.enabled be set @@ -3553,8 +3595,8 @@ spec: agent. properties: enabled: - description: Enabled specifies whether Hubble is enabled - on the agent. + description: Enabled decides if Hubble is enabled on the + agent or not type: boolean metrics: description: Metrics is a list of metrics to collect. @@ -3563,6 +3605,14 @@ spec: type: string type: array type: object + identityAllocationMode: + description: 'IdentityAllocationMode specifies in which backend + identities are stored ("crd", "kvstore"). Default: crd' + type: string + identityChangeGracePeriod: + description: 'IdentityChangeGracePeriod specifies the duration + to wait before using a changed identity. Default: 5s' + type: string ipam: description: Ipam specifies the IP address allocation mode to use. Possible values are "crd" and "eni". "eni" will @@ -3570,7 +3620,8 @@ spec: to be set to false. "crd" will use CRDs for controlling IP address management. "hostscope" will use hostscope IPAM mode. "kubernetes" will use addersing based on node pod - CIDR. Empty value will use host-scope address management. + CIDR. Empty value will use hostscope for cilum <= 1.7 and + "kubernetes" otherwise. type: string ipv4ClusterCidrMaskSize: description: Ipv4ClusterCIDRMaskSize is not implemented and @@ -3760,7 +3811,7 @@ spec: classic: description: ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes. Support been removed - since kubernetes 1.4. + since Kubernetes 1.4. type: object cni: description: CNINetworkingSpec is the specification for networking @@ -3811,7 +3862,7 @@ spec: type: object lyftvpc: description: LyftVPCNetworkingSpec declares that we want to use - the cni-ipvlan-vpc-k8s CNI networking + the cni-ipvlan-vpc-k8s CNI networking. properties: subnetTags: additionalProperties: diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 986b9832ed..7c88b01d75 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -303,6 +303,9 @@ type CiliumNetworkingSpec struct { // DisableConntrack is not implemented and may be removed in the future. // Setting this has no effect. DisableConntrack bool `json:"disableConntrack,omitempty"` + // DisableEndpointCRD disables usage of CiliumEndpoint CRD. + // Default: false + DisableEndpointCRD bool `json:"disableEndpointCRD,omitempty"` // DisableIpv4 is deprecated: Use EnableIpv4 instead. // Setting this flag has no effect. DisableIpv4 bool `json:"disableIpv4,omitempty"` @@ -315,6 +318,15 @@ type CiliumNetworkingSpec struct { // "never": Cilium allows all traffic regardless of policies in place. // If unspecified, "default" policy mode will be used. EnablePolicy string `json:"enablePolicy,omitempty"` + // EnableL7Proxy enables L7 proxy for L7 policy enforcement. + // Default: true + EnableL7Proxy *bool `json:"enableL7Proxy,omitempty"` + // EnableBPFMasquerade enables masquerading packets from endpoints leaving the host with BPF instead of iptables. + // Default: false + EnableBPFMasquerade *bool `json:"enableBPFMasquerade,omitempty"` + // EnableEndpointHealthChecking enables connectivity health checking between virtual endpoints. + // Default: true + EnableEndpointHealthChecking *bool `json:"enableEndpointHealthChecking,omitempty"` // EnableTracing is not implemented and may be removed in the future. // Setting this has no effect. EnableTracing bool `json:"enableTracing,omitempty"` @@ -326,6 +338,12 @@ type CiliumNetworkingSpec struct { // EnvoyLog is not implemented and may be removed in the future. // Setting this has no effect. EnvoyLog string `json:"envoyLog,omitempty"` + // IdentityAllocationMode specifies in which backend identities are stored ("crd", "kvstore"). + // Default: crd + IdentityAllocationMode string `json:"identityAllocationMode,omitempty"` + // IdentityChangeGracePeriod specifies the duration to wait before using a changed identity. + // Default: 5s + IdentityChangeGracePeriod string `json:"identityChangeGracePeriod,omitempty"` // Ipv4ClusterCIDRMaskSize is not implemented and may be removed in the future. // Setting this has no effect. Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` @@ -436,6 +454,24 @@ type CiliumNetworkingSpec struct { // BPFCTGlobalAnyMax is the maximum number of entries in the non-TCP CT table. // Default: 262144 BPFCTGlobalAnyMax int `json:"bpfCTGlobalAnyMax,omitempty"` + // BPFLBAlgorithm is the load balancing algorithm ("random", "maglev"). + // Default: random + BPFLBAlgorithm string `json:"bpfLBAlgorithm,omitempty"` + // BPFLBMaglevTableSize is the per service backend table size when going with Maglev (parameter M). + // Default: 16381 + BPFLBMaglevTableSize string `json:"bpfLBMaglevTableSize,omitempty"` + // BPFNATGlobalMax is the the maximum number of entries in the BPF NAT table. + // Default: 524288 + BPFNATGlobalMax int `json:"bpfNATGlobalMax,omitempty"` + // BPFNeighGlobalMax is the the maximum number of entries in the BPF Neighbor table. + // Default: 524288 + BPFNeighGlobalMax int `json:"bpfNeighGlobalMax,omitempty"` + // BPFPolicyMapMax is the maximum number of entries in endpoint policy map. + // Default: 16384 + BPFPolicyMapMax int `json:"bpfPolicyMapMax,omitempty"` + // BPFLBMapMax is the maximum number of entries in bpf lb service, backend and affinity maps. + // Default: 65536 + BPFLBMapMax int `json:"bpfLBMapMax,omitempty"` // PreallocateBPFMaps reduces the per-packet latency at the expense of up-front memory allocation. // Default: true PreallocateBPFMaps bool `json:"preallocateBPFMaps,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index fa8ee5e1c2..e15655b40f 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -38,7 +38,7 @@ type NetworkingSpec struct { } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes. -// Support been removed since kubernetes 1.4. +// Support been removed since Kubernetes 1.4. type ClassicNetworkingSpec struct { } @@ -256,6 +256,8 @@ type AmazonVPCNetworkingSpec struct { Env []EnvVar `json:"env,omitempty"` } +const CiliumIpamEni = "eni" + // CiliumNetworkingSpec declares that we want Cilium networking type CiliumNetworkingSpec struct { // Version is the version of the Cilium agent and the Cilium Operator. @@ -301,6 +303,9 @@ type CiliumNetworkingSpec struct { // DisableConntrack is not implemented and may be removed in the future. // Setting this has no effect. DisableConntrack bool `json:"disableConntrack,omitempty"` + // DisableEndpointCRD disables usage of CiliumEndpoint CRD. + // Default: false + DisableEndpointCRD bool `json:"disableEndpointCRD,omitempty"` // DisableIpv4 is deprecated: Use EnableIpv4 instead. // Setting this flag has no effect. DisableIpv4 bool `json:"disableIpv4,omitempty"` @@ -313,6 +318,15 @@ type CiliumNetworkingSpec struct { // "never": Cilium allows all traffic regardless of policies in place. // If unspecified, "default" policy mode will be used. EnablePolicy string `json:"enablePolicy,omitempty"` + // EnableL7Proxy enables L7 proxy for L7 policy enforcement. + // Default: true + EnableL7Proxy *bool `json:"enableL7Proxy,omitempty"` + // EnableBPFMasquerade enables masquerading packets from endpoints leaving the host with BPF instead of iptables. + // Default: false + EnableBPFMasquerade *bool `json:"enableBPFMasquerade,omitempty"` + // EnableEndpointHealthChecking enables connectivity health checking between virtual endpoints. + // Default: true + EnableEndpointHealthChecking *bool `json:"enableEndpointHealthChecking,omitempty"` // EnableTracing is not implemented and may be removed in the future. // Setting this has no effect. EnableTracing bool `json:"enableTracing,omitempty"` @@ -324,6 +338,12 @@ type CiliumNetworkingSpec struct { // EnvoyLog is not implemented and may be removed in the future. // Setting this has no effect. EnvoyLog string `json:"envoyLog,omitempty"` + // IdentityAllocationMode specifies in which backend identities are stored ("crd", "kvstore"). + // Default: crd + IdentityAllocationMode string `json:"identityAllocationMode,omitempty"` + // IdentityChangeGracePeriod specifies the duration to wait before using a changed identity. + // Default: 5s + IdentityChangeGracePeriod string `json:"identityChangeGracePeriod,omitempty"` // Ipv4ClusterCIDRMaskSize is not implemented and may be removed in the future. // Setting this has no effect. Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` @@ -434,6 +454,24 @@ type CiliumNetworkingSpec struct { // BPFCTGlobalAnyMax is the maximum number of entries in the non-TCP CT table. // Default: 262144 BPFCTGlobalAnyMax int `json:"bpfCTGlobalAnyMax,omitempty"` + // BPFLBAlgorithm is the load balancing algorithm ("random", "maglev"). + // Default: random + BPFLBAlgorithm string `json:"bpfLBAlgorithm,omitempty"` + // BPFLBMaglevTableSize is the per service backend table size when going with Maglev (parameter M). + // Default: 16381 + BPFLBMaglevTableSize string `json:"bpfLBMaglevTableSize,omitempty"` + // BPFNATGlobalMax is the the maximum number of entries in the BPF NAT table. + // Default: 524288 + BPFNATGlobalMax int `json:"bpfNATGlobalMax,omitempty"` + // BPFNeighGlobalMax is the the maximum number of entries in the BPF Neighbor table. + // Default: 524288 + BPFNeighGlobalMax int `json:"bpfNeighGlobalMax,omitempty"` + // BPFPolicyMapMax is the maximum number of entries in endpoint policy map. + // Default: 16384 + BPFPolicyMapMax int `json:"bpfPolicyMapMax,omitempty"` + // BPFLBMapMax is the maximum number of entries in bpf lb service, backend and affinity maps. + // Default: 65536 + BPFLBMapMax int `json:"bpfLBMapMax,omitempty"` // PreallocateBPFMaps reduces the per-packet latency at the expense of up-front memory allocation. // Default: true PreallocateBPFMaps bool `json:"preallocateBPFMaps,omitempty"` @@ -463,7 +501,7 @@ type CiliumNetworkingSpec struct { // "crd" will use CRDs for controlling IP address management. // "hostscope" will use hostscope IPAM mode. // "kubernetes" will use addersing based on node pod CIDR. - // Empty value will use host-scope address management. + // Empty value will use hostscope for cilum <= 1.7 and "kubernetes" otherwise. Ipam string `json:"ipam,omitempty"` // IPTablesRulesNoinstall disables installing the base IPTables rules used for masquerading and kube-proxy. // Default: false @@ -509,7 +547,7 @@ type CiliumNetworkingSpec struct { // HubbleSpec configures the Hubble service on the Cilium agent. type HubbleSpec struct { - // Enabled specifies whether Hubble is enabled on the agent. + // Enabled decides if Hubble is enabled on the agent or not Enabled *bool `json:"enabled,omitempty"` // Metrics is a list of metrics to collect. If empty or null, metrics are disabled. @@ -517,7 +555,7 @@ type HubbleSpec struct { Metrics []string `json:"metrics,omitempty"` } -// LyftVPCNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking +// LyftVPCNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking. type LyftVPCNetworkingSpec struct { SubnetTags map[string]string `json:"subnetTags,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index e78d6f7302..412043fcd2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1742,13 +1742,19 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.DebugVerbose = in.DebugVerbose out.Device = in.Device out.DisableConntrack = in.DisableConntrack + out.DisableEndpointCRD = in.DisableEndpointCRD out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnableL7Proxy = in.EnableL7Proxy + out.EnableBPFMasquerade = in.EnableBPFMasquerade + out.EnableEndpointHealthChecking = in.EnableEndpointHealthChecking out.EnableTracing = in.EnableTracing out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableEncryption = in.EnableEncryption out.EnvoyLog = in.EnvoyLog + out.IdentityAllocationMode = in.IdentityAllocationMode + out.IdentityChangeGracePeriod = in.IdentityChangeGracePeriod out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node out.Ipv4Range = in.Ipv4Range @@ -1786,6 +1792,12 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.MonitorAggregation = in.MonitorAggregation out.BPFCTGlobalTCPMax = in.BPFCTGlobalTCPMax out.BPFCTGlobalAnyMax = in.BPFCTGlobalAnyMax + out.BPFLBAlgorithm = in.BPFLBAlgorithm + out.BPFLBMaglevTableSize = in.BPFLBMaglevTableSize + out.BPFNATGlobalMax = in.BPFNATGlobalMax + out.BPFNeighGlobalMax = in.BPFNeighGlobalMax + out.BPFPolicyMapMax = in.BPFPolicyMapMax + out.BPFLBMapMax = in.BPFLBMapMax out.PreallocateBPFMaps = in.PreallocateBPFMaps out.SidecarIstioProxyImage = in.SidecarIstioProxyImage out.ClusterName = in.ClusterName @@ -1837,13 +1849,19 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.DebugVerbose = in.DebugVerbose out.Device = in.Device out.DisableConntrack = in.DisableConntrack + out.DisableEndpointCRD = in.DisableEndpointCRD out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnableL7Proxy = in.EnableL7Proxy + out.EnableBPFMasquerade = in.EnableBPFMasquerade + out.EnableEndpointHealthChecking = in.EnableEndpointHealthChecking out.EnableTracing = in.EnableTracing out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableEncryption = in.EnableEncryption out.EnvoyLog = in.EnvoyLog + out.IdentityAllocationMode = in.IdentityAllocationMode + out.IdentityChangeGracePeriod = in.IdentityChangeGracePeriod out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node out.Ipv4Range = in.Ipv4Range @@ -1881,6 +1899,12 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.MonitorAggregation = in.MonitorAggregation out.BPFCTGlobalTCPMax = in.BPFCTGlobalTCPMax out.BPFCTGlobalAnyMax = in.BPFCTGlobalAnyMax + out.BPFLBAlgorithm = in.BPFLBAlgorithm + out.BPFLBMaglevTableSize = in.BPFLBMaglevTableSize + out.BPFNATGlobalMax = in.BPFNATGlobalMax + out.BPFNeighGlobalMax = in.BPFNeighGlobalMax + out.BPFPolicyMapMax = in.BPFPolicyMapMax + out.BPFLBMapMax = in.BPFLBMapMax out.PreallocateBPFMaps = in.PreallocateBPFMaps out.SidecarIstioProxyImage = in.SidecarIstioProxyImage out.ClusterName = in.ClusterName diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 34e57757f3..2d12661e06 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -504,6 +504,21 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.EnableL7Proxy != nil { + in, out := &in.EnableL7Proxy, &out.EnableL7Proxy + *out = new(bool) + **out = **in + } + if in.EnableBPFMasquerade != nil { + in, out := &in.EnableBPFMasquerade, &out.EnableBPFMasquerade + *out = new(bool) + **out = **in + } + if in.EnableEndpointHealthChecking != nil { + in, out := &in.EnableEndpointHealthChecking, &out.EnableEndpointHealthChecking + *out = new(bool) + **out = **in + } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make([]string, len(*in)) diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index b1247903b8..e5321e0517 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -851,6 +851,22 @@ func validateNetworkingCilium(cluster *kops.Cluster, v *kops.CiliumNetworkingSpe allErrs = append(allErrs, IsValidValue(fldPath.Child("containerRuntimeLabels"), &v.ContainerRuntimeLabels, []string{"none", "containerd", "crio", "docker", "auto"})...) } + if v.IdentityAllocationMode != "" { + allErrs = append(allErrs, IsValidValue(fldPath.Child("identityAllocationMode"), &v.IdentityAllocationMode, []string{"crd", "kvstore"})...) + + if v.IdentityAllocationMode == "kvstore" && !v.EtcdManaged { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("identityAllocationMode"), "Cilium requires managed etcd to allocate identities on kvstore mode")) + } + } + + if v.BPFLBAlgorithm != "" { + allErrs = append(allErrs, IsValidValue(fldPath.Child("bpfLBAlgorithm"), &v.BPFLBAlgorithm, []string{"random", "maglev"})...) + } + + if fi.BoolValue(v.EnableL7Proxy) && v.IPTablesRulesNoinstall { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("enableL7Proxy"), "Cilium L7 Proxy requires IPTablesRules to be installed")) + } + if v.Ipam != "" { // "azure" not supported by kops allErrs = append(allErrs, IsValidValue(fldPath.Child("ipam"), &v.Ipam, []string{"hostscope", "kubernetes", "crd", "eni"})...) diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index 6f12aedf2d..7e502b4796 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -821,6 +821,16 @@ func Test_Validate_Cilium(t *testing.T) { }, ExpectedErrors: []string{"Forbidden::cilium.disableMasquerade"}, }, + { + Cilium: kops.CiliumNetworkingSpec{ + EnableL7Proxy: fi.Bool(true), + IPTablesRulesNoinstall: true, + }, + Spec: kops.ClusterSpec{ + CloudProvider: "aws", + }, + ExpectedErrors: []string{"Forbidden::cilium.enableL7Proxy"}, + }, { Cilium: kops.CiliumNetworkingSpec{ Ipam: "eni", @@ -830,6 +840,12 @@ func Test_Validate_Cilium(t *testing.T) { }, ExpectedErrors: []string{"Forbidden::cilium.ipam"}, }, + { + Cilium: kops.CiliumNetworkingSpec{ + IdentityAllocationMode: "kvstore", + }, + ExpectedErrors: []string{"Forbidden::cilium.identityAllocationMode"}, + }, { Cilium: kops.CiliumNetworkingSpec{ Version: "v1.0.0", diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index b8a613002b..3a94dee76b 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -588,6 +588,21 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.EnableL7Proxy != nil { + in, out := &in.EnableL7Proxy, &out.EnableL7Proxy + *out = new(bool) + **out = **in + } + if in.EnableBPFMasquerade != nil { + in, out := &in.EnableBPFMasquerade, &out.EnableBPFMasquerade + *out = new(bool) + **out = **in + } + if in.EnableEndpointHealthChecking != nil { + in, out := &in.EnableEndpointHealthChecking, &out.EnableEndpointHealthChecking + *out = new(bool) + **out = **in + } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make([]string, len(*in)) diff --git a/pkg/model/components/cilium.go b/pkg/model/components/cilium.go index 4e80e16c96..db3c0f91aa 100644 --- a/pkg/model/components/cilium.go +++ b/pkg/model/components/cilium.go @@ -42,14 +42,51 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error { c.Version = "v1.10.0" } + if c.EnableEndpointHealthChecking == nil { + c.EnableEndpointHealthChecking = fi.Bool(true) + } + + if c.IdentityAllocationMode == "" { + c.IdentityAllocationMode = "crd" + } + + if c.IdentityChangeGracePeriod == "" { + c.IdentityChangeGracePeriod = "5s" + } + if c.BPFCTGlobalAnyMax == 0 { c.BPFCTGlobalAnyMax = 262144 } + if c.BPFCTGlobalTCPMax == 0 { c.BPFCTGlobalTCPMax = 524288 } + if c.BPFLBAlgorithm == "" { + c.BPFLBAlgorithm = "random" + } + + if c.BPFLBMaglevTableSize == "" { + c.BPFLBMaglevTableSize = "16381" + } + + if c.BPFNATGlobalMax == 0 { + c.BPFNATGlobalMax = 524288 + } + + if c.BPFNeighGlobalMax == 0 { + c.BPFNeighGlobalMax = 524288 + } + + if c.BPFPolicyMapMax == 0 { + c.BPFPolicyMapMax = 16384 + } + + if c.BPFLBMapMax == 0 { + c.BPFLBMapMax = 65536 + } + if c.ClusterName == "" { c.ClusterName = "default" } @@ -94,6 +131,14 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error { c.EnableRemoteNodeIdentity = fi.Bool(true) } + if c.EnableBPFMasquerade == nil { + c.EnableBPFMasquerade = fi.Bool(false) + } + + if c.EnableL7Proxy == nil { + c.EnableL7Proxy = fi.Bool(true) + } + if c.CPURequest == nil { defaultCPURequest := resource.MustParse("25m") c.CPURequest = &defaultCPURequest diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template index 14d9d9cf7d..aa6a136f0c 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template @@ -61,7 +61,9 @@ data: # backend. Upgrades from these older cilium versions should continue using # the kvstore by commenting out the identity-allocation-mode below, or # setting it to "kvstore". - identity-allocation-mode: crd + # (default crd) + identity-allocation-mode: "{{ .IdentityAllocationMode }}" + # If you want to run cilium in debug mode change this value to true debug: "{{ .Debug }}" {{ if .EnablePrometheusMetrics }} @@ -103,6 +105,32 @@ data: bpf-ct-global-tcp-max: "{{ .BPFCTGlobalTCPMax }}" bpf-ct-global-any-max: "{{ .BPFCTGlobalAnyMax }}" + # BPF load balancing algorithm ("random", "maglev") (default "random") + bpf-lb-algorithm: "{{ .BPFLBAlgorithm }}" + + # Maglev per service backend table size (parameter M) (default 16381) + bpf-lb-maglev-table-size: "{{ .BPFLBMaglevTableSize }}" + + # bpf-nat-global-max specified the maximum number of entries in the + # BPF NAT table. (default 524288) + bpf-nat-global-max: "{{ .BPFNATGlobalMax }}" + + # bpf-neigh-global-max specified the maximum number of entries in the + # BPF neighbor table. (default 524288) + bpf-neigh-global-max: "{{ .BPFNeighGlobalMax }}" + + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) (default 16384) + bpf-policy-map-max: "{{ .BPFPolicyMapMax }}" + + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. (default 65536) + bpf-lb-map-max: "{{ .BPFLBMapMax }}" + + # enable-bpf-masquerade enables masquerading packets from endpoints leaving + # the host with BPF instead of iptables. (default false) + enable-bpf-masquerade: "{{ .EnableBPFMasquerade }}" + # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; @@ -178,7 +206,7 @@ data: {{ end }} enable-node-port: "{{ .EnableNodePort }}" kube-proxy-replacement: "{{- if .EnableNodePort -}}strict{{- else -}}partial{{- end -}}" - enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity -}}" + {{ with .Ipam }} ipam: {{ . }} {{ if eq . "eni" }} @@ -188,6 +216,18 @@ data: {{ end }} {{ end }} + # Disables usage of CiliumEndpoint CRD + disable-endpoint-crd: "{{ .DisableEndpointCRD }}" + + # Enable connectivity health checking between virtual endpoints (default true) + enable-endpoint-health-checking: "{{ .EnableEndpointHealthChecking }}" + + # Enable use of remote node identity (default false) + enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}" + + # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true) + enable-l7-proxy: "{{ .EnableL7Proxy }}" + {{ if WithDefaultBool .Hubble.Enabled false }} # Enable Hubble gRPC service. enable-hubble: "true" diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template index 0e95af2413..2700b80b3d 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template @@ -61,9 +61,15 @@ data: # backend. Upgrades from these older cilium versions should continue using # the kvstore by commenting out the identity-allocation-mode below, or # setting it to "kvstore". - identity-allocation-mode: crd + # (default crd) + identity-allocation-mode: "{{ .IdentityAllocationMode }}" + + # Time to wait before using new identity on endpoint identity change (default 5s) + identity-change-grace-period: "{{ .IdentityChangeGracePeriod }}" + # If you want to run cilium in debug mode change this value to true debug: "{{ .Debug }}" + {{ if .EnablePrometheusMetrics }} # If you want metrics enabled in all of your Cilium agents, set the port for # which the Cilium agents will have their metrics exposed. @@ -103,6 +109,32 @@ data: bpf-ct-global-tcp-max: "{{ .BPFCTGlobalTCPMax }}" bpf-ct-global-any-max: "{{ .BPFCTGlobalAnyMax }}" + # BPF load balancing algorithm ("random", "maglev") (default "random") + bpf-lb-algorithm: "{{ .BPFLBAlgorithm }}" + + # Maglev per service backend table size (parameter M) (default 16381) + bpf-lb-maglev-table-size: "{{ .BPFLBMaglevTableSize }}" + + # bpf-nat-global-max specified the maximum number of entries in the + # BPF NAT table. (default 524288) + bpf-nat-global-max: "{{ .BPFNATGlobalMax }}" + + # bpf-neigh-global-max specified the maximum number of entries in the + # BPF neighbor table. (default 524288) + bpf-neigh-global-max: "{{ .BPFNeighGlobalMax }}" + + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) (default 16384) + bpf-policy-map-max: "{{ .BPFPolicyMapMax }}" + + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. (default 65536) + bpf-lb-map-max: "{{ .BPFLBMapMax }}" + + # enable-bpf-masquerade enables masquerading packets from endpoints leaving + # the host with BPF instead of iptables. (default false) + enable-bpf-masquerade: "{{ .EnableBPFMasquerade }}" + # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; @@ -178,7 +210,7 @@ data: {{ end }} enable-node-port: "{{ .EnableNodePort }}" kube-proxy-replacement: "{{- if .EnableNodePort -}}strict{{- else -}}partial{{- end -}}" - enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity -}}" + {{ with .Ipam }} ipam: {{ . }} {{ if eq . "eni" }} @@ -188,6 +220,18 @@ data: {{ end }} {{ end }} + # Disables usage of CiliumEndpoint CRD + disable-endpoint-crd: "{{ .DisableEndpointCRD }}" + + # Enable connectivity health checking between virtual endpoints (default true) + enable-endpoint-health-checking: "{{ .EnableEndpointHealthChecking }}" + + # Enable use of remote node identity (default false) + enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}" + + # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true) + enable-l7-proxy: "{{ .EnableL7Proxy }}" + {{ if WithDefaultBool .Hubble.Enabled false }} # Enable Hubble gRPC service. enable-hubble: "true" @@ -563,7 +607,7 @@ spec: value: "true" failureThreshold: 24 periodSeconds: 2 - successThreshold: + successThreshold: livenessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 715bbcf185..f2e7dd2a8d 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -61,7 +61,7 @@ spec: version: 1.17.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 27abb72e8f4dfeae4442a1fb3aa949b8ecc98917 + manifestHash: c65dbc9a45ba4eac03942294626a15f4bb511695 name: networking.cilium.io needsRollingUpdate: all selector: