From 0d12f8bface579b4b4d9719cd5203dfccc5d85e6 Mon Sep 17 00:00:00 2001 From: dntosas Date: Mon, 19 Apr 2021 11:11:39 +0300 Subject: [PATCH 1/2] [docs] Add a note for updating hashes when changing addon manifests Signed-off-by: dntosas --- docs/contributing/adding_a_feature.md | 2 ++ docs/networking/cilium.md | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/contributing/adding_a_feature.md b/docs/contributing/adding_a_feature.md index 9fcffea086..a12ce7c337 100644 --- a/docs/contributing/adding_a_feature.md +++ b/docs/contributing/adding_a_feature.md @@ -87,6 +87,8 @@ Then we conditionally move cilium-operator to masters: {{ '{{ end }}' }} ``` +After changing manifest files remember to run `bash hack/update-expected.sh` in order to get updated [manifestHash](https://github.com/kubernetes/kops/blob/master/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml#L74) values. + ## Configuring kubelet When Cilium is in ENI mode `kubelet` needs to be configured with the local IP address, so that it can distinguish it diff --git a/docs/networking/cilium.md b/docs/networking/cilium.md index a6a088bd42..4f2ab7c441 100644 --- a/docs/networking/cilium.md +++ b/docs/networking/cilium.md @@ -138,6 +138,18 @@ Once the secret has been created, encryption can be enabled by setting `enableEn enableEncryption: true ``` +#### Resources in Cilium +{{ kops_feature_table(kops_added_default='1.21', k8s_min='1.20') }} + +As of kOps 1.20, it is possible to choose your own values for Cilium Agents + Operator. Example: +```yaml + networking: + cilium: + cpuRequest: "25m" + memoryRequest: "128Mi" + cpuLimit: "100m" + memoryLimit: "300Mi" +``` ## Getting help From 55524935d786a4d4464bd6747e3758cfa2e3b944 Mon Sep 17 00:00:00 2001 From: dntosas Date: Mon, 19 Apr 2021 11:13:28 +0300 Subject: [PATCH 2/2] [cilium] Add support for choosing resources Cilium as a CNI is a critical component for the cluster so it would be safe to have some guaranteed resources as well as allowing the users to define them based on their needs. In this commit, we init default requested resources and add the capability of user-defined values. Signed-off-by: dntosas --- docs/networking/cilium.md | 2 -- k8s/crds/kops.k8s.io_clusters.yaml | 16 ++++++++++++++++ pkg/apis/kops/networking.go | 5 +++++ pkg/apis/kops/v1alpha2/networking.go | 5 +++++ .../kops/v1alpha2/zz_generated.conversion.go | 4 ++++ pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go | 10 ++++++++++ pkg/apis/kops/zz_generated.deepcopy.go | 10 ++++++++++ pkg/model/components/cilium.go | 11 +++++++++++ .../k8s-1.12-v1.8.yaml.template | 11 +++++++++-- .../k8s-1.12-v1.9.yaml.template | 12 ++++++++++-- .../networking.cilium.io/k8s-1.12.yaml.template | 8 ++++++++ .../bootstrapchannelbuilder/cilium/manifest.yaml | 2 +- 12 files changed, 89 insertions(+), 7 deletions(-) diff --git a/docs/networking/cilium.md b/docs/networking/cilium.md index 4f2ab7c441..0182e28883 100644 --- a/docs/networking/cilium.md +++ b/docs/networking/cilium.md @@ -147,8 +147,6 @@ As of kOps 1.20, it is possible to choose your own values for Cilium Agents + Op cilium: cpuRequest: "25m" memoryRequest: "128Mi" - cpuLimit: "100m" - memoryLimit: "300Mi" ``` ## Getting help diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index df00c15893..828e5b5957 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -3398,6 +3398,14 @@ spec: fetches information from the container runtime and this field is ignored. Default: none' type: string + cpuRequest: + anyOf: + - type: integer + - type: string + description: 'CPURequest CPU request of Cilium agent + operator + container. (default: 25m)' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true debug: description: Debug runs Cilium in debug mode. type: boolean @@ -3588,6 +3596,14 @@ spec: be removed in the future. Setting this has no effect. format: int32 type: integer + memoryRequest: + anyOf: + - type: integer + - type: string + description: 'MemoryRequest memory request of Cilium agent + + operator container. (default: 128Mi)' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true monitorAggregation: description: 'MonitorAggregation sets the level of packet monitoring. Possible values are "low", "medium", or "maximum". diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 30bc5ff0a7..d1f9aac134 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -261,6 +261,11 @@ type CiliumNetworkingSpec struct { // Version is the version of the Cilium agent and the Cilium Operator. Version string `json:"version,omitempty"` + // MemoryRequest memory request of Cilium agent + operator container. (default: 128Mi) + MemoryRequest *resource.Quantity `json:"memoryRequest,omitempty"` + // CPURequest CPU request of Cilium agent + operator container. (default: 25m) + CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` + // AccessLog is not implemented and may be removed in the future. // Setting this has no effect. AccessLog string `json:"accessLog,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index a66434242a..19079139e1 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -259,6 +259,11 @@ type CiliumNetworkingSpec struct { // Version is the version of the Cilium agent and the Cilium Operator. Version string `json:"version,omitempty"` + // MemoryRequest memory request of Cilium agent + operator container. (default: 128Mi) + MemoryRequest *resource.Quantity `json:"memoryRequest,omitempty"` + // CPURequest CPU request of Cilium agent + operator container. (default: 25m) + CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` + // AccessLog is not implemented and may be removed in the future. // Setting this has no effect. AccessLog string `json:"accessLog,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 258044ae53..4722edf4f5 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1636,6 +1636,8 @@ func Convert_kops_CertManagerConfig_To_v1alpha2_CertManagerConfig(in *kops.CertM func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *CiliumNetworkingSpec, out *kops.CiliumNetworkingSpec, s conversion.Scope) error { out.Version = in.Version + out.MemoryRequest = in.MemoryRequest + out.CPURequest = in.CPURequest out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels out.AgentPrometheusPort = in.AgentPrometheusPort @@ -1728,6 +1730,8 @@ func Convert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *Cili func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *kops.CiliumNetworkingSpec, out *CiliumNetworkingSpec, s conversion.Scope) error { out.Version = in.Version + out.MemoryRequest = in.MemoryRequest + out.CPURequest = in.CPURequest out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels out.AgentPrometheusPort = in.AgentPrometheusPort diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index ac9c91c80d..34e02da3c9 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -431,6 +431,16 @@ func (in *CertManagerConfig) DeepCopy() *CertManagerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) { *out = *in + if in.MemoryRequest != nil { + in, out := &in.MemoryRequest, &out.MemoryRequest + x := (*in).DeepCopy() + *out = &x + } + if in.CPURequest != nil { + in, out := &in.CPURequest, &out.CPURequest + x := (*in).DeepCopy() + *out = &x + } if in.AgentLabels != nil { in, out := &in.AgentLabels, &out.AgentLabels *out = make([]string, len(*in)) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index e13bfbf2bb..9d95368b25 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -531,6 +531,16 @@ func (in *ChannelSpec) DeepCopy() *ChannelSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) { *out = *in + if in.MemoryRequest != nil { + in, out := &in.MemoryRequest, &out.MemoryRequest + x := (*in).DeepCopy() + *out = &x + } + if in.CPURequest != nil { + in, out := &in.CPURequest, &out.CPURequest + x := (*in).DeepCopy() + *out = &x + } if in.AgentLabels != nil { in, out := &in.AgentLabels, &out.AgentLabels *out = make([]string, len(*in)) diff --git a/pkg/model/components/cilium.go b/pkg/model/components/cilium.go index db2c749618..000f28ad65 100644 --- a/pkg/model/components/cilium.go +++ b/pkg/model/components/cilium.go @@ -18,6 +18,7 @@ package components import ( "github.com/blang/semver/v4" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" @@ -92,6 +93,16 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error { c.EnableRemoteNodeIdentity = fi.Bool(true) } + if c.CPURequest == nil { + defaultCPURequest := resource.MustParse("25m") + c.CPURequest = &defaultCPURequest + } + + if c.MemoryRequest == nil { + defaultMemoryRequest := resource.MustParse("128Mi") + c.MemoryRequest = &defaultMemoryRequest + } + hubble := c.Hubble if hubble != nil { if hubble.Enabled == nil { diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template index 436db95bfd..8794464130 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template @@ -522,7 +522,10 @@ spec: protocol: TCP {{- end }} {{ end }} - + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} readinessProbe: httpGet: host: '127.0.0.1' @@ -772,6 +775,10 @@ spec: name: prometheus protocol: TCP {{ end }} + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} livenessProbe: httpGet: host: "127.0.0.1" @@ -915,4 +922,4 @@ spec: path: /var/run/cilium type: Directory name: hubble-sock-dir -{{ end }} \ No newline at end of file +{{ end }} diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template index e41bdaf0d7..be51cf8a5f 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template @@ -525,6 +525,10 @@ spec: periodSeconds: 30 successThreshold: 1 timeoutSeconds: 5 + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} readinessProbe: httpGet: host: '127.0.0.1' @@ -823,6 +827,10 @@ spec: name: prometheus protocol: TCP {{ end }} + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} livenessProbe: httpGet: host: '127.0.0.1' @@ -891,7 +899,7 @@ spec: strategy: rollingUpdate: maxUnavailable: 1 - type: RollingUpdate + type: RollingUpdate template: metadata: labels: @@ -951,4 +959,4 @@ spec: path: config.yaml name: config {{ end }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template index f773689508..028390dab8 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template @@ -461,6 +461,10 @@ spec: name: prometheus protocol: TCP {{ end }} + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} readinessProbe: exec: command: @@ -750,6 +754,10 @@ spec: name: prometheus protocol: TCP {{ end }} + resources: + requests: + cpu: {{ or .CPURequest "25m" }} + memory: {{ or .MemoryRequest "128Mi" }} livenessProbe: httpGet: host: "127.0.0.1" diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 075d692a35..91e09a963f 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -70,7 +70,7 @@ spec: version: 1.17.0 - id: k8s-1.12 manifest: networking.cilium.io/k8s-1.12-v1.9.yaml - manifestHash: a1d86d4d8501a5f4adfc7e6c356377730a507c86 + manifestHash: dea8534ba3aa267f877f7c2f68a1899fe869e1d3 name: networking.cilium.io needsRollingUpdate: all selector: