diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index a28cf72fc1..685a0dcc17 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -742,6 +742,23 @@ spec: logFormat: json ``` +### Graceful Node Shutdown + +Graceful node shutdown allows kubelet to prevent instance shutdown until Pods have been safely terminated or a timeout has been reached. + +For all CNIs except `amazonaws`, kOps will try to add a 30 second timeout for 30 seconds where the first 20 seconds is reserved for normal Pods and the last 10 seconds for critical Pods. When using `amazonaws` this feature is disabled, as it leads to [leaking ENIs](https://github.com/aws/amazon-vpc-cni-k8s/issues/1223). + +This configuration can be changed as follows: + +```yaml +spec: + kubelet: + shutdownGracePeriod: 60s + shutdownGracePeriodCriticalPods: 20s +``` + +Note that Kubelet will fail to install the shutdown inhibtor on systems where logind is configured with an `InhibitDelayMaxSeconds` lower than `shutdownGracePeriod`. On Ubuntu, this setting is 30 seconds. + ## kubeScheduler This block contains configurations for `kube-scheduler`. See https://kubernetes.io/docs/admin/kube-scheduler/ diff --git a/docs/releases/1.23-NOTES.md b/docs/releases/1.23-NOTES.md index e38217a33c..3f5720a3bc 100644 --- a/docs/releases/1.23-NOTES.md +++ b/docs/releases/1.23-NOTES.md @@ -12,6 +12,8 @@ This is a document to gather the release notes prior to the release. being used, then Kubernetes Node resources will be named after their AWS instance ID instead of their domain name and managed subnets will be configured to launch instances with Resource Based Names. +* Support for [ShutdownGracePeriod and ShutdownGracePeriodCriticalPods](https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/). + # Breaking changes * Support for Kubernetes version 1.17 has been removed. diff --git a/go.mod b/go.mod index e1405e2a27..646a453f05 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c k8s.io/klog/v2 v2.30.0 k8s.io/kubectl v0.23.1 + k8s.io/kubelet v0.23.1 k8s.io/legacy-cloud-providers v0.23.1 k8s.io/mount-utils v0.23.1 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b diff --git a/go.sum b/go.sum index 25826c80a5..b04d3f6a51 100644 --- a/go.sum +++ b/go.sum @@ -2066,6 +2066,7 @@ k8s.io/kube-proxy v0.23.1/go.mod h1:65QJpMrjUMHfgX1q5Pl/KqVRZBMM4qLHNMo5MhKsnp0= k8s.io/kube-scheduler v0.23.1/go.mod h1:SFPvXnt7KlxTZILrtjH8VNwGDzXcdKKHrv4TkeZdYro= k8s.io/kubectl v0.23.1 h1:gmscOiV4Y4XIRIn14gQBBADoyyVrDZPbxRCTDga4RSA= k8s.io/kubectl v0.23.1/go.mod h1:Ui7dJKdUludF8yWAOSN7JZEkOuYixX5yF6E6NjoukKE= +k8s.io/kubelet v0.23.1 h1:YBqEOCfSKGWbj1pofkZUpaViFKtAY6gUD8MSXek2/eI= k8s.io/kubelet v0.23.1/go.mod h1:WdvMiehtNPhtiW8sSVVvr8YYU00L0u+0HkfMDEB0LKM= k8s.io/kubernetes v1.23.1/go.mod h1:baMGbPpwwP0kT/+eAPtdqoWNRoXyyTJ2Zf+fw/Y8t04= k8s.io/legacy-cloud-providers v0.23.1 h1:1aoEY/PkwyE3pPk6Yk4mT00CkBZ7w573/LPxtsGiB0s= diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 40527af9cd..1175ae3dc7 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -3732,6 +3732,15 @@ spec: the default value on nodes that // run docker daemon with version < 1.9 or an Aufs storage backend. // Issue #10959 has more details.' type: boolean + shutdownGracePeriod: + description: 'ShutdownGracePeriod specifies the total duration + that the node should delay the shutdown by. Default: 30s' + type: string + shutdownGracePeriodCriticalPods: + description: 'ShutdownGracePeriodCriticalPods specifies the duration + used to terminate critical pods during a node shutdown. Default: + 10s' + type: string streamingConnectionIdleTimeout: description: StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is @@ -4151,6 +4160,15 @@ spec: the default value on nodes that // run docker daemon with version < 1.9 or an Aufs storage backend. // Issue #10959 has more details.' type: boolean + shutdownGracePeriod: + description: 'ShutdownGracePeriod specifies the total duration + that the node should delay the shutdown by. Default: 30s' + type: string + shutdownGracePeriodCriticalPods: + description: 'ShutdownGracePeriodCriticalPods specifies the duration + used to terminate critical pods during a node shutdown. Default: + 10s' + type: string streamingConnectionIdleTimeout: description: StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 39821af52d..4a0b0c4155 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -603,6 +603,15 @@ spec: the default value on nodes that // run docker daemon with version < 1.9 or an Aufs storage backend. // Issue #10959 has more details.' type: boolean + shutdownGracePeriod: + description: 'ShutdownGracePeriod specifies the total duration + that the node should delay the shutdown by. Default: 30s' + type: string + shutdownGracePeriodCriticalPods: + description: 'ShutdownGracePeriodCriticalPods specifies the duration + used to terminate critical pods during a node shutdown. Default: + 10s' + type: string streamingConnectionIdleTimeout: description: StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index 953152bde4..14d2d50f2f 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -75,8 +75,11 @@ go_library( "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/kubelet/config/v1beta1:go_default_library", "//vendor/k8s.io/mount-utils:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 907c204aa2..662b8c2f21 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -17,6 +17,7 @@ limitations under the License. package model import ( + "bytes" "fmt" "net" "os" @@ -32,6 +33,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + kubelet "k8s.io/kubelet/config/v1beta1" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/flagbuilder" @@ -50,6 +55,8 @@ const ( // kubeletService is the name of the kubelet service kubeletService = "kubelet.service" + + kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf" ) // KubeletBuilder installs kubelet @@ -71,6 +78,15 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("error building kubelet config: %v", err) } + { + t, err := buildKubeletComponentConfig(kubeletConfig) + if err != nil { + return err + } + + c.AddTask(t) + } + { t, err := b.buildSystemdEnvironmentFile(kubeletConfig) if err != nil { @@ -199,6 +215,39 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { return nil } +func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { + componentConfig := kubelet.KubeletConfiguration{ + ShutdownGracePeriod: *kubeletConfig.ShutdownGracePeriod, + ShutdownGracePeriodCriticalPods: *kubeletConfig.ShutdownGracePeriodCriticalPods, + } + + s := runtime.NewScheme() + if err := kubelet.AddToScheme(s); err != nil { + return nil, err + } + + gv := kubelet.SchemeGroupVersion + codecFactory := serializer.NewCodecFactory(s) + info, ok := runtime.SerializerInfoForMediaType(codecFactory.SupportedMediaTypes(), "application/yaml") + if !ok { + return nil, fmt.Errorf("failed to find serializer") + } + encoder := codecFactory.EncoderForVersion(info.Serializer, gv) + var w bytes.Buffer + if err := encoder.Encode(&componentConfig, &w); err != nil { + return nil, err + } + + t := &nodetasks.File{ + Path: "/var/lib/kubelet/kubelet.conf", + Contents: fi.NewBytesResource(w.Bytes()), + Type: nodetasks.FileType_File, + BeforeServices: []string{kubeletService}, + } + + return t, nil +} + // kubeletPath returns the path of the kubelet based on distro func (b *KubeletBuilder) kubeletPath() string { kubeletCommand := "/usr/local/bin/kubelet" @@ -286,6 +335,8 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet flags += " --node-ip=::" } + flags += " --config=" + kubeletConfigFilePath + sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n" // Makes kubelet read /root/.docker/config.json properly sysconfig = sysconfig + "HOME=\"/root" + "\"\n" diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index 66039b1fde..f35acc4f22 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -20,8 +20,11 @@ import ( "fmt" "path/filepath" "testing" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/assets" @@ -220,14 +223,23 @@ func runKubeletBuilder(t *testing.T, context *fi.ModelBuilderContext, nodeupMode t.Fatalf("error from KubeletBuilder buildKubeletConfig: %v", err) return } + { + fileTask, err := buildKubeletComponentConfig(kubeletConfig) + if err != nil { + t.Fatalf("error from KubeletBuilder buildKubeletComponentConfig: %v", err) + return + } - fileTask, err := builder.buildSystemdEnvironmentFile(kubeletConfig) - if err != nil { - t.Fatalf("error from KubeletBuilder buildSystemdEnvironmentFile: %v", err) - return + context.AddTask(fileTask) + } + { + fileTask, err := builder.buildSystemdEnvironmentFile(kubeletConfig) + if err != nil { + t.Fatalf("error from KubeletBuilder buildSystemdEnvironmentFile: %v", err) + return + } + context.AddTask(fileTask) } - context.AddTask(fileTask) - { task, err := builder.buildManifestDirectory(kubeletConfig) if err != nil { @@ -415,3 +427,15 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu testutils.ValidateTasks(t, filepath.Join(basedir, "tasks-"+key+".yaml"), context) } + +func Test_BuildComponentConfigFile(t *testing.T) { + componentConfig := kops.KubeletConfigSpec{ + ShutdownGracePeriod: &metav1.Duration{Duration: 30 * time.Second}, + ShutdownGracePeriodCriticalPods: &metav1.Duration{Duration: 10 * time.Second}, + } + + _, err := buildKubeletComponentConfig(&componentConfig) + if err != nil { + t.Errorf("Failed to build component config file: %v", err) + } +} diff --git a/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml b/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml index 2aa710d2bb..4e655e9e75 100644 --- a/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml +++ b/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml @@ -3,11 +3,48 @@ path: /etc/kubernetes/manifests type: directory --- contents: | - DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=AllowExtTrafficLocalEndpoints=false,CSIMigrationAWS=true,ExperimentalCriticalPodAnnotation=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key" + DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=AllowExtTrafficLocalEndpoints=false,CSIMigrationAWS=true,ExperimentalCriticalPodAnnotation=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key --config=/var/lib/kubelet/kubelet.conf" HOME="/root" path: /etc/sysconfig/kubelet type: file --- +beforeServices: +- kubelet.service +contents: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: {} + webhook: + cacheTTL: 0s + x509: {} + authorization: + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cpuManagerReconcilePeriod: 0s + evictionPressureTransitionPeriod: 0s + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + runtimeRequestTimeout: 0s + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 0s +path: /var/lib/kubelet/kubelet.conf +type: file +--- Name: kubelet.service definition: | [Unit] diff --git a/nodeup/pkg/model/tests/kubelet/warmpool/tasks.yaml b/nodeup/pkg/model/tests/kubelet/warmpool/tasks.yaml index 9ca243564a..44552b34b0 100644 --- a/nodeup/pkg/model/tests/kubelet/warmpool/tasks.yaml +++ b/nodeup/pkg/model/tests/kubelet/warmpool/tasks.yaml @@ -3,11 +3,48 @@ path: /etc/kubernetes/manifests type: directory --- contents: | - DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key" + DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key --config=/var/lib/kubelet/kubelet.conf" HOME="/root" path: /etc/sysconfig/kubelet type: file --- +beforeServices: +- kubelet.service +contents: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: {} + webhook: + cacheTTL: 0s + x509: {} + authorization: + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cpuManagerReconcilePeriod: 0s + evictionPressureTransitionPeriod: 0s + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + runtimeRequestTimeout: 0s + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 0s +path: /var/lib/kubelet/kubelet.conf +type: file +--- Name: kubelet.service definition: | [Unit] diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 2a5ac1cbf4..4114374313 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -225,6 +225,13 @@ type KubeletConfigSpec struct { EnableCadvisorJsonEndpoints *bool `json:"enableCadvisorJsonEndpoints,omitempty" flag:"enable-cadvisor-json-endpoints"` // PodPidsLimit is the maximum number of pids in any pod. PodPidsLimit *int64 `json:"podPidsLimit,omitempty" flag:"pod-max-pids"` + + // ShutdownGracePeriod specifies the total duration that the node should delay the shutdown by. + // Default: 30s + ShutdownGracePeriod *metav1.Duration `json:"shutdownGracePeriod,omitempty"` + // ShutdownGracePeriodCriticalPods specifies the duration used to terminate critical pods during a node shutdown. + // Default: 10s + ShutdownGracePeriodCriticalPods *metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` } // KubeProxyConfig defines the configuration for a proxy diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 59d35b6504..1d4610e9a4 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -225,6 +225,13 @@ type KubeletConfigSpec struct { EnableCadvisorJsonEndpoints *bool `json:"enableCadvisorJsonEndpoints,omitempty" flag:"enable-cadvisor-json-endpoints"` // PodPidsLimit is the maximum number of pids in any pod. PodPidsLimit *int64 `json:"podPidsLimit,omitempty" flag:"pod-max-pids"` + + // ShutdownGracePeriod specifies the total duration that the node should delay the shutdown by. + // Default: 30s + ShutdownGracePeriod *metav1.Duration `json:"shutdownGracePeriod,omitempty"` + // ShutdownGracePeriodCriticalPods specifies the duration used to terminate critical pods during a node shutdown. + // Default: 10s + ShutdownGracePeriodCriticalPods *metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` } // KubeProxyConfig defines the configuration for a proxy diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 40d94197e2..e7080e8a02 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -5464,6 +5464,8 @@ func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.ContainerLogMaxFiles = in.ContainerLogMaxFiles out.EnableCadvisorJsonEndpoints = in.EnableCadvisorJsonEndpoints out.PodPidsLimit = in.PodPidsLimit + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods return nil } @@ -5561,6 +5563,8 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha2_KubeletConfigSpec(in *kops.K out.ContainerLogMaxFiles = in.ContainerLogMaxFiles out.EnableCadvisorJsonEndpoints = in.EnableCadvisorJsonEndpoints out.PodPidsLimit = in.PodPidsLimit + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 7fa55a2b58..2e77cf495c 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -3760,6 +3760,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = new(int64) **out = **in } + if in.ShutdownGracePeriod != nil { + in, out := &in.ShutdownGracePeriod, &out.ShutdownGracePeriod + *out = new(v1.Duration) + **out = **in + } + if in.ShutdownGracePeriodCriticalPods != nil { + in, out := &in.ShutdownGracePeriodCriticalPods, &out.ShutdownGracePeriodCriticalPods + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index 6ae1b5ebc4..14249a3923 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -223,6 +223,13 @@ type KubeletConfigSpec struct { EnableCadvisorJsonEndpoints *bool `json:"enableCadvisorJsonEndpoints,omitempty" flag:"enable-cadvisor-json-endpoints"` // PodPidsLimit is the maximum number of pids in any pod. PodPidsLimit *int64 `json:"podPidsLimit,omitempty" flag:"pod-max-pids"` + + // ShutdownGracePeriod specifies the total duration that the node should delay the shutdown by. + // Default: 30s + ShutdownGracePeriod *metav1.Duration `json:"shutdownGracePeriod,omitempty"` + // ShutdownGracePeriodCriticalPods specifies the duration used to terminate critical pods during a node shutdown. + // Default: 10s + ShutdownGracePeriodCriticalPods *metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` } // KubeProxyConfig defines the configuration for a proxy diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 698d757793..90e47acd52 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -5352,6 +5352,8 @@ func autoConvert_v1alpha3_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.ContainerLogMaxFiles = in.ContainerLogMaxFiles out.EnableCadvisorJsonEndpoints = in.EnableCadvisorJsonEndpoints out.PodPidsLimit = in.PodPidsLimit + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods return nil } @@ -5449,6 +5451,8 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha3_KubeletConfigSpec(in *kops.K out.ContainerLogMaxFiles = in.ContainerLogMaxFiles out.EnableCadvisorJsonEndpoints = in.EnableCadvisorJsonEndpoints out.PodPidsLimit = in.PodPidsLimit + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index f85a3f66c0..d870402c82 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -3666,6 +3666,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = new(int64) **out = **in } + if in.ShutdownGracePeriod != nil { + in, out := &in.ShutdownGracePeriod, &out.ShutdownGracePeriod + *out = new(v1.Duration) + **out = **in + } + if in.ShutdownGracePeriodCriticalPods != nil { + in, out := &in.ShutdownGracePeriodCriticalPods, &out.ShutdownGracePeriodCriticalPods + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index adede9cf5a..42bb72d22d 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -713,6 +713,14 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi } } + if k.ShutdownGracePeriodCriticalPods != nil { + if k.ShutdownGracePeriod == nil { + allErrs = append(allErrs, field.Forbidden(kubeletPath.Child("shutdownGracePeriodCriticalPods"), "shutdownGracePeriodCriticalPods require shutdownGracePeriod")) + } + if k.ShutdownGracePeriod.Duration.Seconds() < k.ShutdownGracePeriodCriticalPods.Seconds() { + allErrs = append(allErrs, field.Invalid(kubeletPath.Child("shutdownGracePeriodCriticalPods"), k.ShutdownGracePeriodCriticalPods.String(), "shutdownGracePeriodCriticalPods cannot be greater than shutdownGracePeriod")) + } + } } return allErrs } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d166e08233..c73ccc4fc9 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -3845,6 +3845,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = new(int64) **out = **in } + if in.ShutdownGracePeriod != nil { + in, out := &in.ShutdownGracePeriod, &out.ShutdownGracePeriod + *out = new(v1.Duration) + **out = **in + } + if in.ShutdownGracePeriodCriticalPods != nil { + in, out := &in.ShutdownGracePeriodCriticalPods, &out.ShutdownGracePeriodCriticalPods + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/model/components/kubelet.go b/pkg/model/components/kubelet.go index bb28638766..5d536186eb 100644 --- a/pkg/model/components/kubelet.go +++ b/pkg/model/components/kubelet.go @@ -19,7 +19,9 @@ package components import ( "fmt" "strings" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" @@ -207,5 +209,14 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec.Kubelet.ProtectKernelDefaults = fi.Bool(true) } + // We do not enable graceful shutdown when using amazonaws due to leaking ENIs. + if clusterSpec.Kubelet.ShutdownGracePeriod == nil && clusterSpec.Networking.AmazonVPC == nil { + clusterSpec.Kubelet.ShutdownGracePeriod = &metav1.Duration{Duration: time.Duration(30 * time.Second)} + clusterSpec.Kubelet.ShutdownGracePeriodCriticalPods = &metav1.Duration{Duration: time.Duration(10 * time.Second)} + } else if clusterSpec.Networking.AmazonVPC != nil { + clusterSpec.Kubelet.ShutdownGracePeriod = &metav1.Duration{Duration: 0} + clusterSpec.Kubelet.ShutdownGracePeriodCriticalPods = &metav1.Duration{Duration: 0} + } + return nil } diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 2099e3c254..c037d2247f 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -2463,6 +2463,7 @@ k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2R k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kubectl v0.22.2/go.mod h1:BApg2j0edxLArCOfO0ievI27EeTQqBDMNU9VQH734iQ= +k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/legacy-cloud-providers v0.22.2/go.mod h1:oC6zhm9nhJ5M4VTDHzsO/4MpddZR5JqEt55zZ52JRMc= k8s.io/metrics v0.22.2/go.mod h1:GUcsBtpsqQD1tKFS/2wCKu4ZBowwRncLOJH1rgWs3uw= diff --git a/vendor/k8s.io/kubelet/LICENSE b/vendor/k8s.io/kubelet/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/k8s.io/kubelet/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/kubelet/config/v1beta1/BUILD.bazel b/vendor/k8s.io/kubelet/config/v1beta1/BUILD.bazel new file mode 100644 index 0000000000..802df3dcfa --- /dev/null +++ b/vendor/k8s.io/kubelet/config/v1beta1/BUILD.bazel @@ -0,0 +1,21 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "doc.go", + "register.go", + "types.go", + "zz_generated.deepcopy.go", + ], + importmap = "k8s.io/kops/vendor/k8s.io/kubelet/config/v1beta1", + importpath = "k8s.io/kubelet/config/v1beta1", + visibility = ["//visibility:public"], + deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/component-base/config/v1alpha1:go_default_library", + ], +) diff --git a/vendor/k8s.io/kubelet/config/v1beta1/doc.go b/vendor/k8s.io/kubelet/config/v1beta1/doc.go new file mode 100644 index 0000000000..5644377d6c --- /dev/null +++ b/vendor/k8s.io/kubelet/config/v1beta1/doc.go @@ -0,0 +1,21 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +groupName=kubelet.config.k8s.io + +package v1beta1 // import "k8s.io/kubelet/config/v1beta1" diff --git a/vendor/k8s.io/kubelet/config/v1beta1/register.go b/vendor/k8s.io/kubelet/config/v1beta1/register.go new file mode 100644 index 0000000000..914dfb25fa --- /dev/null +++ b/vendor/k8s.io/kubelet/config/v1beta1/register.go @@ -0,0 +1,44 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name used in this package +const GroupName = "kubelet.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} + +var ( + // SchemeBuilder is the scheme builder with scheme init functions to run for this API package + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme is a global function that registers this API group & version to a scheme + AddToScheme = SchemeBuilder.AddToScheme +) + +// addKnownTypes registers known types to the given scheme +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &KubeletConfiguration{}, + &SerializedNodeConfigSource{}, + ) + return nil +} diff --git a/vendor/k8s.io/kubelet/config/v1beta1/types.go b/vendor/k8s.io/kubelet/config/v1beta1/types.go new file mode 100644 index 0000000000..223a88576c --- /dev/null +++ b/vendor/k8s.io/kubelet/config/v1beta1/types.go @@ -0,0 +1,1183 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" +) + +// HairpinMode denotes how the kubelet should configure networking to handle +// hairpin packets. +type HairpinMode string + +// Enum settings for different ways to handle hairpin packets. +const ( + // Set the hairpin flag on the veth of containers in the respective + // container runtime. + HairpinVeth = "hairpin-veth" + // Make the container bridge promiscuous. This will force it to accept + // hairpin packets, even if the flag isn't set on ports of the bridge. + PromiscuousBridge = "promiscuous-bridge" + // Neither of the above. If the kubelet is started in this hairpin mode + // and kube-proxy is running in iptables mode, hairpin packets will be + // dropped by the container bridge. + HairpinNone = "none" +) + +// ResourceChangeDetectionStrategy denotes a mode in which internal +// managers (secret, configmap) are discovering object changes. +type ResourceChangeDetectionStrategy string + +// Enum settings for different strategies of kubelet managers. +const ( + // GetChangeDetectionStrategy is a mode in which kubelet fetches + // necessary objects directly from apiserver. + GetChangeDetectionStrategy ResourceChangeDetectionStrategy = "Get" + // TTLCacheChangeDetectionStrategy is a mode in which kubelet uses + // ttl cache for object directly fetched from apiserver. + TTLCacheChangeDetectionStrategy ResourceChangeDetectionStrategy = "Cache" + // WatchChangeDetectionStrategy is a mode in which kubelet uses + // watches to observe changes to objects that are in its interest. + WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch" + // RestrictedTopologyManagerPolicy is a mode in which kubelet only allows + // pods with optimal NUMA node alignment for requested resources + RestrictedTopologyManagerPolicy = "restricted" + // BestEffortTopologyManagerPolicy is a mode in which kubelet will favour + // pods with NUMA alignment of CPU and device resources. + BestEffortTopologyManagerPolicy = "best-effort" + // NoneTopologyManagerPolicy is a mode in which kubelet has no knowledge + // of NUMA alignment of a pod's CPU and device resources. + NoneTopologyManagerPolicy = "none" + // SingleNumaNodeTopologyManagerPolicy is a mode in which kubelet only allows + // pods with a single NUMA alignment of CPU and device resources. + SingleNumaNodeTopologyManagerPolicy = "single-numa-node" + // ContainerTopologyManagerScope represents that + // topology policy is applied on a per-container basis. + ContainerTopologyManagerScope = "container" + // PodTopologyManagerScope represents that + // topology policy is applied on a per-pod basis. + PodTopologyManagerScope = "pod" + // NoneMemoryManagerPolicy is a memory manager none policy, under the none policy + // the memory manager will not pin containers memory of guaranteed pods + NoneMemoryManagerPolicy = "None" + // StaticMemoryManagerPolicy is a memory manager static policy, under the static policy + // the memory manager will try to pin containers memory of guaranteed pods to the smallest + // possible sub-set of NUMA nodes + StaticMemoryManagerPolicy = "Static" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KubeletConfiguration contains the configuration for the Kubelet +type KubeletConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // enableServer enables Kubelet's secured server. + // Note: Kubelet's insecure port is controlled by the readOnlyPort option. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: true + EnableServer *bool `json:"enableServer,omitempty"` + // staticPodPath is the path to the directory containing local (static) pods to + // run, or the path to a single static pod file. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // the set of static pods specified at the new path may be different than the + // ones the Kubelet initially started with, and this may disrupt your node. + // Default: "" + // +optional + StaticPodPath string `json:"staticPodPath,omitempty"` + // syncFrequency is the max period between synchronizing running + // containers and config. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // shortening this duration may have a negative performance impact, especially + // as the number of Pods on the node increases. Alternatively, increasing this + // duration will result in longer refresh times for ConfigMaps and Secrets. + // Default: "1m" + // +optional + SyncFrequency metav1.Duration `json:"syncFrequency,omitempty"` + // fileCheckFrequency is the duration between checking config files for + // new data. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // shortening the duration will cause the Kubelet to reload local Static Pod + // configurations more frequently, which may have a negative performance impact. + // Default: "20s" + // +optional + FileCheckFrequency metav1.Duration `json:"fileCheckFrequency,omitempty"` + // httpCheckFrequency is the duration between checking http for new data. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // shortening the duration will cause the Kubelet to poll staticPodURL more + // frequently, which may have a negative performance impact. + // Default: "20s" + // +optional + HTTPCheckFrequency metav1.Duration `json:"httpCheckFrequency,omitempty"` + // staticPodURL is the URL for accessing static pods to run. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // the set of static pods specified at the new URL may be different than the + // ones the Kubelet initially started with, and this may disrupt your node. + // Default: "" + // +optional + StaticPodURL string `json:"staticPodURL,omitempty"` + // staticPodURLHeader is a map of slices with HTTP headers to use when accessing the podURL. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt the ability to read the latest set of static pods from StaticPodURL. + // Default: nil + // +optional + StaticPodURLHeader map[string][]string `json:"staticPodURLHeader,omitempty"` + // address is the IP address for the Kubelet to serve on (set to 0.0.0.0 + // for all interfaces). + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: "0.0.0.0" + // +optional + Address string `json:"address,omitempty"` + // port is the port for the Kubelet to serve on. + // The port number must be between 1 and 65535, inclusive. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: 10250 + // +optional + Port int32 `json:"port,omitempty"` + // readOnlyPort is the read-only port for the Kubelet to serve on with + // no authentication/authorization. + // The port number must be between 1 and 65535, inclusive. + // Setting this field to 0 disables the read-only service. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: 0 (disabled) + // +optional + ReadOnlyPort int32 `json:"readOnlyPort,omitempty"` + // tlsCertFile is the file containing x509 Certificate for HTTPS. (CA cert, + // if any, concatenated after server cert). If tlsCertFile and + // tlsPrivateKeyFile are not provided, a self-signed certificate + // and key are generated for the public address and saved to the directory + // passed to the Kubelet's --cert-dir flag. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: "" + // +optional + TLSCertFile string `json:"tlsCertFile,omitempty"` + // tlsPrivateKeyFile is the file containing x509 private key matching tlsCertFile. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: "" + // +optional + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty"` + // tlsCipherSuites is the list of allowed cipher suites for the server. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: nil + // +optional + TLSCipherSuites []string `json:"tlsCipherSuites,omitempty"` + // tlsMinVersion is the minimum TLS version supported. + // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: "" + // +optional + TLSMinVersion string `json:"tlsMinVersion,omitempty"` + // rotateCertificates enables client certificate rotation. The Kubelet will request a + // new certificate from the certificates.k8s.io API. This requires an approver to approve the + // certificate signing requests. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // disabling it may disrupt the Kubelet's ability to authenticate with the API server + // after the current certificate expires. + // Default: false + // +optional + RotateCertificates bool `json:"rotateCertificates,omitempty"` + // serverTLSBootstrap enables server certificate bootstrap. Instead of self + // signing a serving certificate, the Kubelet will request a certificate from + // the 'certificates.k8s.io' API. This requires an approver to approve the + // certificate signing requests (CSR). The RotateKubeletServerCertificate feature + // must be enabled when setting this field. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // disabling it will stop the renewal of Kubelet server certificates, which can + // disrupt components that interact with the Kubelet server in the long term, + // due to certificate expiration. + // Default: false + // +optional + ServerTLSBootstrap bool `json:"serverTLSBootstrap,omitempty"` + // authentication specifies how requests to the Kubelet's server are authenticated. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Defaults: + // anonymous: + // enabled: false + // webhook: + // enabled: true + // cacheTTL: "2m" + // +optional + Authentication KubeletAuthentication `json:"authentication"` + // authorization specifies how requests to the Kubelet's server are authorized. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Defaults: + // mode: Webhook + // webhook: + // cacheAuthorizedTTL: "5m" + // cacheUnauthorizedTTL: "30s" + // +optional + Authorization KubeletAuthorization `json:"authorization"` + // registryPullQPS is the limit of registry pulls per second. + // The value must not be a negative number. + // Setting it to 0 means no limit. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic produced + // by image pulls. + // Default: 5 + // +optional + RegistryPullQPS *int32 `json:"registryPullQPS,omitempty"` + // registryBurst is the maximum size of bursty pulls, temporarily allows + // pulls to burst to this number, while still not exceeding registryPullQPS. + // The value must not be a negative number. + // Only used if registryPullQPS is greater than 0. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic produced + // by image pulls. + // Default: 10 + // +optional + RegistryBurst int32 `json:"registryBurst,omitempty"` + // eventRecordQPS is the maximum event creations per second. If 0, there + // is no limit enforced. The value cannot be a negative number. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic produced by + // event creations. + // Default: 5 + // +optional + EventRecordQPS *int32 `json:"eventRecordQPS,omitempty"` + // eventBurst is the maximum size of a burst of event creations, temporarily + // allows event creations to burst to this number, while still not exceeding + // eventRecordQPS. This field canot be a negative number and it is only used + // when eventRecordQPS > 0. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic produced by + // event creations. + // Default: 10 + // +optional + EventBurst int32 `json:"eventBurst,omitempty"` + // enableDebuggingHandlers enables server endpoints for log access + // and local running of containers and commands, including the exec, + // attach, logs, and portforward features. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // disabling it may disrupt components that interact with the Kubelet server. + // Default: true + // +optional + EnableDebuggingHandlers *bool `json:"enableDebuggingHandlers,omitempty"` + // enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // enabling it may carry a performance impact. + // Default: false + // +optional + EnableContentionProfiling bool `json:"enableContentionProfiling,omitempty"` + // healthzPort is the port of the localhost healthz endpoint (set to 0 to disable). + // A valid number is between 1 and 65535. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that monitor Kubelet health. + // Default: 10248 + // +optional + HealthzPort *int32 `json:"healthzPort,omitempty"` + // healthzBindAddress is the IP address for the healthz server to serve on. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that monitor Kubelet health. + // Default: "127.0.0.1" + // +optional + HealthzBindAddress string `json:"healthzBindAddress,omitempty"` + // oomScoreAdj is The oom-score-adj value for kubelet process. Values + // must be within the range [-1000, 1000]. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the stability of nodes under memory pressure. + // Default: -999 + // +optional + OOMScoreAdj *int32 `json:"oomScoreAdj,omitempty"` + // clusterDomain is the DNS domain for this cluster. If set, kubelet will + // configure all containers to search this domain in addition to the + // host's search domains. + // Dynamic Kubelet Config (deprecated): Dynamically updating this field is not recommended, + // as it should be kept in sync with the rest of the cluster. + // Default: "" + // +optional + ClusterDomain string `json:"clusterDomain,omitempty"` + // clusterDNS is a list of IP addresses for the cluster DNS server. If set, + // kubelet will configure all containers to use this for DNS resolution + // instead of the host's DNS servers. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // changes will only take effect on Pods created after the update. Draining + // the node is recommended before changing this field. + // Default: nil + // +optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // streamingConnectionIdleTimeout is the maximum time a streaming connection + // can be idle before the connection is automatically closed. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact components that rely on infrequent updates over streaming + // connections to the Kubelet server. + // Default: "4h" + // +optional + StreamingConnectionIdleTimeout metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty"` + // nodeStatusUpdateFrequency is the frequency that kubelet computes node + // status. If node lease feature is not enabled, it is also the frequency that + // kubelet posts node status to master. + // Note: When node lease feature is not enabled, be cautious when changing the + // constant, it must work with nodeMonitorGracePeriod in nodecontroller. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact node scalability, and also that the node controller's + // nodeMonitorGracePeriod must be set to N*NodeStatusUpdateFrequency, + // where N is the number of retries before the node controller marks + // the node unhealthy. + // Default: "10s" + // +optional + NodeStatusUpdateFrequency metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty"` + // nodeStatusReportFrequency is the frequency that kubelet posts node + // status to master if node status does not change. Kubelet will ignore this + // frequency and post node status immediately if any change is detected. It is + // only used when node lease feature is enabled. nodeStatusReportFrequency's + // default value is 5m. But if nodeStatusUpdateFrequency is set explicitly, + // nodeStatusReportFrequency's default value will be set to + // nodeStatusUpdateFrequency for backward compatibility. + // Default: "5m" + // +optional + NodeStatusReportFrequency metav1.Duration `json:"nodeStatusReportFrequency,omitempty"` + // nodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease. + // NodeLease provides an indicator of node health by having the Kubelet create and + // periodically renew a lease, named after the node, in the kube-node-lease namespace. + // If the lease expires, the node can be considered unhealthy. + // The lease is currently renewed every 10s, per KEP-0009. In the future, the lease renewal + // interval may be set based on the lease duration. + // The field value must be greater than 0. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // decreasing the duration may reduce tolerance for issues that temporarily prevent + // the Kubelet from renewing the lease (e.g. a short-lived network issue). + // Default: 40 + // +optional + NodeLeaseDurationSeconds int32 `json:"nodeLeaseDurationSeconds,omitempty"` + // imageMinimumGCAge is the minimum age for an unused image before it is + // garbage collected. If DynamicKubeletConfig (deprecated; default off) + // is on, when dynamically updating this field, consider that it may trigger or + // delay garbage collection, and may change the image overhead on the node. + // Default: "2m" + // +optional + ImageMinimumGCAge metav1.Duration `json:"imageMinimumGCAge,omitempty"` + // imageGCHighThresholdPercent is the percent of disk usage after which + // image garbage collection is always run. The percent is calculated by + // dividing this field value by 100, so this field must be between 0 and + // 100, inclusive. When specified, the value must be greater than + // imageGCLowThresholdPercent. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger or delay garbage collection, and may change the image overhead + // on the node. + // Default: 85 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // imageGCLowThresholdPercent is the percent of disk usage before which + // image garbage collection is never run. Lowest disk usage to garbage + // collect to. The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. When specified, the + // value must be less than imageGCHighThresholdPercent. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger or delay garbage collection, and may change the image overhead + // on the node. + // Default: 80 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // volumeStatsAggPeriod is the frequency for calculating and caching volume + // disk usage for all pods. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // shortening the period may carry a performance impact. + // Default: "1m" + // +optional + VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod,omitempty"` + // kubeletCgroups is the absolute name of cgroups to isolate the kubelet in + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "" + // +optional + KubeletCgroups string `json:"kubeletCgroups,omitempty"` + // systemCgroups is absolute name of cgroups in which to place + // all non-kernel processes that are not already in a container. Empty + // for no container. Rolling back the flag requires a reboot. + // The cgroupRoot must be specified if this field is not empty. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "" + // +optional + SystemCgroups string `json:"systemCgroups,omitempty"` + // cgroupRoot is the root cgroup to use for pods. This is handled by the + // container runtime on a best effort basis. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "" + // +optional + CgroupRoot string `json:"cgroupRoot,omitempty"` + // cgroupsPerQOS enable QoS based CGroup hierarchy: top level CGroups for QoS classes + // and all Burstable and BestEffort Pods are brought up under their specific top level + // QoS CGroup. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: true + // +optional + CgroupsPerQOS *bool `json:"cgroupsPerQOS,omitempty"` + // cgroupDriver is the driver kubelet uses to manipulate CGroups on the host (cgroupfs + // or systemd). + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "cgroupfs" + // +optional + CgroupDriver string `json:"cgroupDriver,omitempty"` + // cpuManagerPolicy is the name of the policy to use. + // Requires the CPUManager feature gate to be enabled. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "None" + // +optional + CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"` + // cpuManagerPolicyOptions is a set of key=value which allows to set extra options + // to fine tune the behaviour of the cpu manager policies. + // Requires both the "CPUManager" and "CPUManagerPolicyOptions" feature gates to be enabled. + // Dynamic Kubelet Config (beta): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: nil + // +optional + CPUManagerPolicyOptions map[string]string `json:"cpuManagerPolicyOptions,omitempty"` + // cpuManagerReconcilePeriod is the reconciliation period for the CPU Manager. + // Requires the CPUManager feature gate to be enabled. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // shortening the period may carry a performance impact. + // Default: "10s" + // +optional + CPUManagerReconcilePeriod metav1.Duration `json:"cpuManagerReconcilePeriod,omitempty"` + // memoryManagerPolicy is the name of the policy to use by memory manager. + // Requires the MemoryManager feature gate to be enabled. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "none" + // +optional + MemoryManagerPolicy string `json:"memoryManagerPolicy,omitempty"` + // topologyManagerPolicy is the name of the topology manager policy to use. + // Valid values include: + // + // - `restricted`: kubelet only allows pods with optimal NUMA node alignment for + // requested resources; + // - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device + // resources; + // - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + // - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + // of CPU and device resources. + // + // Policies other than "none" require the TopologyManager feature gate to be enabled. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "none" + // +optional + TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"` + // topologyManagerScope represents the scope of topology hint generation + // that topology manager requests and hint providers generate. Valid values include: + // + // - `container`: topology policy is applied on a per-container basis. + // - `pod`: topology policy is applied on a per-pod basis. + // + // "pod" scope requires the TopologyManager feature gate to be enabled. + // Default: "container" + // +optional + TopologyManagerScope string `json:"topologyManagerScope,omitempty"` + // qosReserved is a set of resource name to percentage pairs that specify + // the minimum percentage of a resource reserved for exclusive use by the + // guaranteed QoS tier. + // Currently supported resources: "memory" + // Requires the QOSReserved feature gate to be enabled. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: nil + // +optional + QOSReserved map[string]string `json:"qosReserved,omitempty"` + // runtimeRequestTimeout is the timeout for all runtime requests except long running + // requests - pull, logs, exec and attach. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may disrupt components that interact with the Kubelet server. + // Default: "2m" + // +optional + RuntimeRequestTimeout metav1.Duration `json:"runtimeRequestTimeout,omitempty"` + // hairpinMode specifies how the Kubelet should configure the container + // bridge for hairpin packets. + // Setting this flag allows endpoints in a Service to loadbalance back to + // themselves if they should try to access their own Service. Values: + // + // - "promiscuous-bridge": make the container bridge promiscuous. + // - "hairpin-veth": set the hairpin flag on container veth interfaces. + // - "none": do nothing. + // + // Generally, one must set `--hairpin-mode=hairpin-veth to` achieve hairpin NAT, + // because promiscuous-bridge assumes the existence of a container bridge named cbr0. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may require a node reboot, depending on the network plugin. + // Default: "promiscuous-bridge" + // +optional + HairpinMode string `json:"hairpinMode,omitempty"` + // maxPods is the maximum number of Pods that can run on this Kubelet. + // The value must be a non-negative integer. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // changes may cause Pods to fail admission on Kubelet restart, and may change + // the value reported in Node.Status.Capacity[v1.ResourcePods], thus affecting + // future scheduling decisions. Increasing this value may also decrease performance, + // as more Pods can be packed into a single node. + // Default: 110 + // +optional + MaxPods int32 `json:"maxPods,omitempty"` + // podCIDR is the CIDR to use for pod IP addresses, only used in standalone mode. + // In cluster mode, this is obtained from the control plane. + // Dynamic Kubelet Config (deprecated): This field should always be set to the empty default. + // It should only set for standalone Kubelets, which cannot use Dynamic Kubelet Config. + // Default: "" + // +optional + PodCIDR string `json:"podCIDR,omitempty"` + // podPidsLimit is the maximum number of PIDs in any pod. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // lowering it may prevent container processes from forking after the change. + // Default: -1 + // +optional + PodPidsLimit *int64 `json:"podPidsLimit,omitempty"` + // resolvConf is the resolver configuration file used as the basis + // for the container DNS resolution configuration. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // changes will only take effect on Pods created after the update. Draining + // the node is recommended before changing this field. + // If set to the empty string, will override the default and effectively disable DNS lookups. + // Default: "/etc/resolv.conf" + // +optional + ResolverConfig *string `json:"resolvConf,omitempty"` + // runOnce causes the Kubelet to check the API server once for pods, + // run those in addition to the pods specified by static pod files, and exit. + // Default: false + // +optional + RunOnce bool `json:"runOnce,omitempty"` + // cpuCFSQuota enables CPU CFS quota enforcement for containers that + // specify CPU limits. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // disabling it may reduce node stability. + // Default: true + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` + // cpuCFSQuotaPeriod is the CPU CFS quota period value, `cpu.cfs_period_us`. + // The value must be between 1 us and 1 second, inclusive. + // Requires the CustomCPUCFSQuotaPeriod feature gate to be enabled. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // limits set for containers will result in different cpu.cfs_quota settings. This + // will trigger container restarts on the node being reconfigured. + // Default: "100ms" + // +optional + CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty"` + // nodeStatusMaxImages caps the number of images reported in Node.status.images. + // The value must be greater than -2. + // Note: If -1 is specified, no cap will be applied. If 0 is specified, no image is returned. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // different values can be reported on node status. + // Default: 50 + // +optional + NodeStatusMaxImages *int32 `json:"nodeStatusMaxImages,omitempty"` + // maxOpenFiles is Number of files that can be opened by Kubelet process. + // The value must be a non-negative number. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the ability of the Kubelet to interact with the node's filesystem. + // Default: 1000000 + // +optional + MaxOpenFiles int64 `json:"maxOpenFiles,omitempty"` + // contentType is contentType of requests sent to apiserver. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the ability for the Kubelet to communicate with the API server. + // If the Kubelet loses contact with the API server due to a change to this field, + // the change cannot be reverted via dynamic Kubelet config. + // Default: "application/vnd.kubernetes.protobuf" + // +optional + ContentType string `json:"contentType,omitempty"` + // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic the Kubelet + // sends to the API server. + // Default: 5 + // +optional + KubeAPIQPS *int32 `json:"kubeAPIQPS,omitempty"` + // kubeAPIBurst is the burst to allow while talking with kubernetes API server. + // This field cannot be a negative number. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact scalability by changing the amount of traffic the Kubelet + // sends to the API server. + // Default: 10 + // +optional + KubeAPIBurst int32 `json:"kubeAPIBurst,omitempty"` + // serializeImagePulls when enabled, tells the Kubelet to pull images one + // at a time. We recommend *not* changing the default value on nodes that + // run docker daemon with version < 1.9 or an Aufs storage backend. + // Issue #10959 has more details. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the performance of image pulls. + // Default: true + // +optional + SerializeImagePulls *bool `json:"serializeImagePulls,omitempty"` + // evictionHard is a map of signal names to quantities that defines hard eviction + // thresholds. For example: `{"memory.available": "300Mi"}`. + // To explicitly disable, pass a 0% or 100% threshold on an arbitrary resource. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger or delay Pod evictions. + // Default: + // memory.available: "100Mi" + // nodefs.available: "10%" + // nodefs.inodesFree: "5%" + // imagefs.available: "15%" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // evictionSoft is a map of signal names to quantities that defines soft eviction thresholds. + // For example: `{"memory.available": "300Mi"}`. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger or delay Pod evictions, and may change the allocatable reported + // by the node. + // Default: nil + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // evictionSoftGracePeriod is a map of signal names to quantities that defines grace + // periods for each soft eviction signal. For example: `{"memory.available": "30s"}`. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger or delay Pod evictions. + // Default: nil + // +optional + EvictionSoftGracePeriod map[string]string `json:"evictionSoftGracePeriod,omitempty"` + // evictionPressureTransitionPeriod is the duration for which the kubelet has to wait + // before transitioning out of an eviction pressure condition. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // lowering it may decrease the stability of the node when the node is overcommitted. + // Default: "5m" + // +optional + EvictionPressureTransitionPeriod metav1.Duration `json:"evictionPressureTransitionPeriod,omitempty"` + // evictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use + // when terminating pods in response to a soft eviction threshold being met. This value + // effectively caps the Pod's terminationGracePeriodSeconds value during soft evictions. + // Note: Due to issue #64530, the behavior has a bug where this value currently just + // overrides the grace period during soft eviction, which can increase the grace + // period from what is set on the Pod. This bug will be fixed in a future release. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // lowering it decreases the amount of time Pods will have to gracefully clean + // up before being killed during a soft eviction. + // Default: 0 + // +optional + EvictionMaxPodGracePeriod int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // evictionMinimumReclaim is a map of signal names to quantities that defines minimum reclaims, + // which describe the minimum amount of a given resource the kubelet will reclaim when + // performing a pod eviction while that resource is under pressure. + // For example: `{"imagefs.available": "2Gi"}`. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may change how well eviction can manage resource pressure. + // Default: nil + // +optional + EvictionMinimumReclaim map[string]string `json:"evictionMinimumReclaim,omitempty"` + // podsPerCore is the maximum number of pods per core. Cannot exceed maxPods. + // The value must be a non-negative integer. + // If 0, there is no limit on the number of Pods. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // changes may cause Pods to fail admission on Kubelet restart, and may change + // the value reported in `Node.status.capacity.pods`, thus affecting + // future scheduling decisions. Increasing this value may also decrease performance, + // as more Pods can be packed into a single node. + // Default: 0 + // +optional + PodsPerCore int32 `json:"podsPerCore,omitempty"` + // enableControllerAttachDetach enables the Attach/Detach controller to + // manage attachment/detachment of volumes scheduled to this node, and + // disables kubelet from executing any attach/detach operations. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // changing which component is responsible for volume management on a live node + // may result in volumes refusing to detach if the node is not drained prior to + // the update, and if Pods are scheduled to the node before the + // volumes.kubernetes.io/controller-managed-attach-detach annotation is updated by the + // Kubelet. In general, it is safest to leave this value set the same as local config. + // Default: true + // +optional + EnableControllerAttachDetach *bool `json:"enableControllerAttachDetach,omitempty"` + // protectKernelDefaults, if true, causes the Kubelet to error if kernel + // flags are not as it expects. Otherwise the Kubelet will attempt to modify + // kernel flags to match its expectation. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // enabling it may cause the Kubelet to crash-loop if the Kernel is not configured as + // Kubelet expects. + // Default: false + // +optional + ProtectKernelDefaults bool `json:"protectKernelDefaults,omitempty"` + // makeIPTablesUtilChains, if true, causes the Kubelet ensures a set of iptables rules + // are present on host. + // These rules will serve as utility rules for various components, e.g. kube-proxy. + // The rules will be created based on iptablesMasqueradeBit and iptablesDropBit. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // disabling it will prevent the Kubelet from healing locally misconfigured iptables rules. + // Default: true + // +optional + MakeIPTablesUtilChains *bool `json:"makeIPTablesUtilChains,omitempty"` + // iptablesMasqueradeBit is the bit of the iptables fwmark space to mark for SNAT. + // Values must be within the range [0, 31]. Must be different from other mark bits. + // Warning: Please match the value of the corresponding parameter in kube-proxy. + // TODO: clean up IPTablesMasqueradeBit in kube-proxy. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it needs to be coordinated with other components, like kube-proxy, and the update + // will only be effective if MakeIPTablesUtilChains is enabled. + // Default: 14 + // +optional + IPTablesMasqueradeBit *int32 `json:"iptablesMasqueradeBit,omitempty"` + // iptablesDropBit is the bit of the iptables fwmark space to mark for dropping packets. + // Values must be within the range [0, 31]. Must be different from other mark bits. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it needs to be coordinated with other components, like kube-proxy, and the update + // will only be effective if MakeIPTablesUtilChains is enabled. + // Default: 15 + // +optional + IPTablesDropBit *int32 `json:"iptablesDropBit,omitempty"` + // featureGates is a map of feature names to bools that enable or disable experimental + // features. This field modifies piecemeal the built-in default values from + // "k8s.io/kubernetes/pkg/features/kube_features.go". + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider the + // documentation for the features you are enabling or disabling. While we + // encourage feature developers to make it possible to dynamically enable + // and disable features, some changes may require node reboots, and some + // features may require careful coordination to retroactively disable. + // Default: nil + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` + // failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // setting it to true will cause the Kubelet to crash-loop if swap is enabled. + // Default: true + // +optional + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // memorySwap configures swap memory available to container workloads. + // +featureGate=NodeSwap + // +optional + MemorySwap MemorySwapConfiguration `json:"memorySwap,omitempty"` + // containerLogMaxSize is a quantity defining the maximum size of the container log + // file before it is rotated. For example: "5Mi" or "256Ki". + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may trigger log rotation. + // Default: "10Mi" + // +optional + ContainerLogMaxSize string `json:"containerLogMaxSize,omitempty"` + // containerLogMaxFiles specifies the maximum number of container log files that can + // be present for a container. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // lowering it may cause log files to be deleted. + // Default: 5 + // +optional + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // configMapAndSecretChangeDetectionStrategy is a mode in which ConfigMap and Secret + // managers are running. Valid values include: + // + // - `Get`: kubelet fetches necessary objects directly from the API server; + // - `Cache`: kubelet uses TTL cache for object fetched from the API server; + // - `Watch`: kubelet uses watches to observe changes to objects that are in its interest. + // + // Default: "Watch" + // +optional + ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy `json:"configMapAndSecretChangeDetectionStrategy,omitempty"` + + /* the following fields are meant for Node Allocatable */ + + // systemReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) + // pairs that describe resources reserved for non-kubernetes components. + // Currently only cpu and memory are supported. + // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may not be possible to increase the reserved resources, because this + // requires resizing cgroups. Always look for a NodeAllocatableEnforced event + // after updating this field to ensure that the update was successful. + // Default: nil + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // kubeReserved is a set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs + // that describe resources reserved for kubernetes system components. + // Currently cpu, memory and local storage for root file system are supported. + // See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + // for more details. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may not be possible to increase the reserved resources, because this + // requires resizing cgroups. Always look for a NodeAllocatableEnforced event + // after updating this field to ensure that the update was successful. + // Default: nil + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // The reservedSystemCPUs option specifies the CPU list reserved for the host + // level system threads and kubernetes related threads. This provide a "static" + // CPU list rather than the "dynamic" list by systemReserved and kubeReserved. + // This option does not support systemReservedCgroup or kubeReservedCgroup. + ReservedSystemCPUs string `json:"reservedSystemCPUs,omitempty"` + // showHiddenMetricsForVersion is the previous version for which you want to show + // hidden metrics. + // Only the previous minor version is meaningful, other values will not be allowed. + // The format is `.`, e.g.: `1.16`. + // The purpose of this format is make sure you have the opportunity to notice + // if the next release hides additional metrics, rather than being surprised + // when they are permanently removed in the release after that. + // Default: "" + // +optional + ShowHiddenMetricsForVersion string `json:"showHiddenMetricsForVersion,omitempty"` + // systemReservedCgroup helps the kubelet identify absolute name of top level CGroup used + // to enforce `systemReserved` compute resource reservation for OS system daemons. + // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) + // doc for more information. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "" + // +optional + SystemReservedCgroup string `json:"systemReservedCgroup,omitempty"` + // kubeReservedCgroup helps the kubelet identify absolute name of top level CGroup used + // to enforce `KubeReserved` compute resource reservation for Kubernetes node system daemons. + // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) + // doc for more information. + // Dynamic Kubelet Config (deprecated): This field should not be updated without a full node + // reboot. It is safest to keep this value the same as the local config. + // Default: "" + // +optional + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + // This flag specifies the various Node Allocatable enforcements that Kubelet needs to perform. + // This flag accepts a list of options. Acceptable options are `none`, `pods`, + // `system-reserved` and `kube-reserved`. + // If `none` is specified, no other options may be specified. + // When `system-reserved` is in the list, systemReservedCgroup must be specified. + // When `kube-reserved` is in the list, kubeReservedCgroup must be specified. + // This field is supported only when `cgroupsPerQOS` is set to true. + // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) + // for more information. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // removing enforcements may reduce the stability of the node. Alternatively, adding + // enforcements may reduce the stability of components which were using more than + // the reserved amount of resources; for example, enforcing kube-reserved may cause + // Kubelets to OOM if it uses more than the reserved resources, and enforcing system-reserved + // may cause system daemons to OOM if they use more than the reserved resources. + // Default: ["pods"] + // +optional + EnforceNodeAllocatable []string `json:"enforceNodeAllocatable,omitempty"` + // A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `*`). + // Unsafe sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, + // and `net.*`. For example: "`kernel.msg*,net.ipv4.route.min_pmtu`" + // Default: [] + // +optional + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` + // volumePluginDir is the full path of the directory in which to search + // for additional third party volume plugins. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that changing + // the volumePluginDir may disrupt workloads relying on third party volume plugins. + // Default: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" + // +optional + VolumePluginDir string `json:"volumePluginDir,omitempty"` + // providerID, if set, sets the unique ID of the instance that an external + // provider (i.e. cloudprovider) can use to identify a specific node. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the ability of the Kubelet to interact with cloud providers. + // Default: "" + // +optional + ProviderID string `json:"providerID,omitempty"` + // kernelMemcgNotification, if set, instructs the the kubelet to integrate with the + // kernel memcg notification for determining if memory eviction thresholds are + // exceeded rather than polling. + // If DynamicKubeletConfig (deprecated; default off) is on, when + // dynamically updating this field, consider that + // it may impact the way Kubelet interacts with the kernel. + // Default: false + // +optional + KernelMemcgNotification bool `json:"kernelMemcgNotification,omitempty"` + // logging specifies the options of logging. + // Refer to [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) + // for more information. + // Default: + // Format: text + // + optional + Logging componentbaseconfigv1alpha1.LoggingConfiguration `json:"logging,omitempty"` + // enableSystemLogHandler enables system logs via web interface host:port/logs/ + // Default: true + // +optional + EnableSystemLogHandler *bool `json:"enableSystemLogHandler,omitempty"` + // shutdownGracePeriod specifies the total duration that the node should delay the + // shutdown and total grace period for pod termination during a node shutdown. + // Default: "0s" + // +featureGate=GracefulNodeShutdown + // +optional + ShutdownGracePeriod metav1.Duration `json:"shutdownGracePeriod,omitempty"` + // shutdownGracePeriodCriticalPods specifies the duration used to terminate critical + // pods during a node shutdown. This should be less than shutdownGracePeriod. + // For example, if shutdownGracePeriod=30s, and shutdownGracePeriodCriticalPods=10s, + // during a node shutdown the first 20 seconds would be reserved for gracefully + // terminating normal pods, and the last 10 seconds would be reserved for terminating + // critical pods. + // Default: "0s" + // +featureGate=GracefulNodeShutdown + // +optional + ShutdownGracePeriodCriticalPods metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` + // shutdownGracePeriodByPodPriority specifies the shutdown grace period for Pods based + // on their associated priority class value. + // When a shutdown request is received, the Kubelet will initiate shutdown on all pods + // running on the node with a grace period that depends on the priority of the pod, + // and then wait for all pods to exit. + // Each entry in the array represents the graceful shutdown time a pod with a priority + // class value that lies in the range of that value and the next higher entry in the + // list when the node is shutting down. + // For example, to allow critical pods 10s to shutdown, priority>=10000 pods 20s to + // shutdown, and all remaining pods 30s to shutdown. + // + // shutdownGracePeriodByPodPriority: + // - priority: 2000000000 + // shutdownGracePeriodSeconds: 10 + // - priority: 10000 + // shutdownGracePeriodSeconds: 20 + // - priority: 0 + // shutdownGracePeriodSeconds: 30 + // + // The time the Kubelet will wait before exiting will at most be the maximum of all + // shutdownGracePeriodSeconds for each priority class range represented on the node. + // When all pods have exited or reached their grace periods, the Kubelet will release + // the shutdown inhibit lock. + // Requires the GracefulNodeShutdown feature gate to be enabled. + // This configuration must be empty if either ShutdownGracePeriod or ShutdownGracePeriodCriticalPods is set. + // Default: nil + // +featureGate=GracefulNodeShutdownBasedOnPodPriority + // +optional + ShutdownGracePeriodByPodPriority []ShutdownGracePeriodByPodPriority `json:"shutdownGracePeriodByPodPriority,omitempty"` + // reservedMemory specifies a comma-separated list of memory reservations for NUMA nodes. + // The parameter makes sense only in the context of the memory manager feature. + // The memory manager will not allocate reserved memory for container workloads. + // For example, if you have a NUMA0 with 10Gi of memory and the reservedMemory was + // specified to reserve 1Gi of memory at NUMA0, the memory manager will assume that + // only 9Gi is available for allocation. + // You can specify a different amount of NUMA node and memory types. + // You can omit this parameter at all, but you should be aware that the amount of + // reserved memory from all NUMA nodes should be equal to the amount of memory specified + // by the [node allocatable](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable). + // If at least one node allocatable parameter has a non-zero value, you will need + // to specify at least one NUMA node. + // Also, avoid specifying: + // + // 1. Duplicates, the same NUMA node, and memory type, but with a different value. + // 2. zero limits for any memory type. + // 3. NUMAs nodes IDs that do not exist under the machine. + // 4. memory types except for memory and hugepages- + // + // Default: nil + // +optional + ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"` + // enableProfilingHandler enables profiling via web interface host:port/debug/pprof/ + // Default: true + // +optional + EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"` + // enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v + // Default: true + // +optional + EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"` + // SeccompDefault enables the use of `RuntimeDefault` as the default seccomp profile for all workloads. + // This requires the corresponding SeccompDefault feature gate to be enabled as well. + // Default: false + // +optional + SeccompDefault *bool `json:"seccompDefault,omitempty"` + // MemoryThrottlingFactor specifies the factor multiplied by the memory limit or node allocatable memory + // when setting the cgroupv2 memory.high value to enforce MemoryQoS. + // Decreasing this factor will set lower high limit for container cgroups and put heavier reclaim pressure + // while increasing will put less reclaim pressure. + // See http://kep.k8s.io/2570 for more details. + // Default: 0.8 + // +featureGate=MemoryQoS + // +optional + MemoryThrottlingFactor *float64 `json:"memoryThrottlingFactor,omitempty"` + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + // Default: nil + // +optional + RegisterWithTaints []v1.Taint `json:"registerWithTaints,omitempty"` + // registerNode enables automatic registration with the apiserver. + // Default: true + // +optional + RegisterNode *bool `json:"registerNode,omitempty"` +} + +type KubeletAuthorizationMode string + +const ( + // KubeletAuthorizationModeAlwaysAllow authorizes all authenticated requests + KubeletAuthorizationModeAlwaysAllow KubeletAuthorizationMode = "AlwaysAllow" + // KubeletAuthorizationModeWebhook uses the SubjectAccessReview API to determine authorization + KubeletAuthorizationModeWebhook KubeletAuthorizationMode = "Webhook" +) + +type KubeletAuthorization struct { + // mode is the authorization mode to apply to requests to the kubelet server. + // Valid values are `AlwaysAllow` and `Webhook`. + // Webhook mode uses the SubjectAccessReview API to determine authorization. + // +optional + Mode KubeletAuthorizationMode `json:"mode,omitempty"` + + // webhook contains settings related to Webhook authorization. + // +optional + Webhook KubeletWebhookAuthorization `json:"webhook"` +} + +type KubeletWebhookAuthorization struct { + // cacheAuthorizedTTL is the duration to cache 'authorized' responses from the + // webhook authorizer. + // +optional + CacheAuthorizedTTL metav1.Duration `json:"cacheAuthorizedTTL,omitempty"` + // cacheUnauthorizedTTL is the duration to cache 'unauthorized' responses from + // the webhook authorizer. + // +optional + CacheUnauthorizedTTL metav1.Duration `json:"cacheUnauthorizedTTL,omitempty"` +} + +type KubeletAuthentication struct { + // x509 contains settings related to x509 client certificate authentication. + // +optional + X509 KubeletX509Authentication `json:"x509"` + // webhook contains settings related to webhook bearer token authentication. + // +optional + Webhook KubeletWebhookAuthentication `json:"webhook"` + // anonymous contains settings related to anonymous authentication. + // +optional + Anonymous KubeletAnonymousAuthentication `json:"anonymous"` +} + +type KubeletX509Authentication struct { + // clientCAFile is the path to a PEM-encoded certificate bundle. If set, any request + // presenting a client certificate signed by one of the authorities in the bundle + // is authenticated with a username corresponding to the CommonName, + // and groups corresponding to the Organization in the client certificate. + // +optional + ClientCAFile string `json:"clientCAFile,omitempty"` +} + +type KubeletWebhookAuthentication struct { + // enabled allows bearer token authentication backed by the + // tokenreviews.authentication.k8s.io API. + // +optional + Enabled *bool `json:"enabled,omitempty"` + // cacheTTL enables caching of authentication results + // +optional + CacheTTL metav1.Duration `json:"cacheTTL,omitempty"` +} + +type KubeletAnonymousAuthentication struct { + // enabled allows anonymous requests to the kubelet server. + // Requests that are not rejected by another authentication method are treated as + // anonymous requests. + // Anonymous requests have a username of `system:anonymous`, and a group name of + // `system:unauthenticated`. + // +optional + Enabled *bool `json:"enabled,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SerializedNodeConfigSource allows us to serialize v1.NodeConfigSource. +// This type is used internally by the Kubelet for tracking checkpointed dynamic configs. +// It exists in the kubeletconfig API group because it is classified as a versioned input to the Kubelet. +type SerializedNodeConfigSource struct { + metav1.TypeMeta `json:",inline"` + // source is the source that we are serializing. + // +optional + Source v1.NodeConfigSource `json:"source,omitempty" protobuf:"bytes,1,opt,name=source"` +} + +// MemoryReservation specifies the memory reservation of different types for each NUMA node +type MemoryReservation struct { + NumaNode int32 `json:"numaNode"` + Limits v1.ResourceList `json:"limits"` +} + +// ShutdownGracePeriodByPodPriority specifies the shutdown grace period for Pods based on their associated priority class value +type ShutdownGracePeriodByPodPriority struct { + // priority is the priority value associated with the shutdown grace period + Priority int32 `json:"priority"` + // shutdownGracePeriodSeconds is the shutdown grace period in seconds + ShutdownGracePeriodSeconds int64 `json:"shutdownGracePeriodSeconds"` +} + +type MemorySwapConfiguration struct { + // swapBehavior configures swap memory available to container workloads. May be one of + // "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit + // "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit. + // +featureGate=NodeSwap + // +optional + SwapBehavior string `json:"swapBehavior,omitempty"` +} diff --git a/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..777c058850 --- /dev/null +++ b/vendor/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,521 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1beta1 + +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAnonymousAuthentication) DeepCopyInto(out *KubeletAnonymousAuthentication) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAnonymousAuthentication. +func (in *KubeletAnonymousAuthentication) DeepCopy() *KubeletAnonymousAuthentication { + if in == nil { + return nil + } + out := new(KubeletAnonymousAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAuthentication) DeepCopyInto(out *KubeletAuthentication) { + *out = *in + out.X509 = in.X509 + in.Webhook.DeepCopyInto(&out.Webhook) + in.Anonymous.DeepCopyInto(&out.Anonymous) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthentication. +func (in *KubeletAuthentication) DeepCopy() *KubeletAuthentication { + if in == nil { + return nil + } + out := new(KubeletAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletAuthorization) DeepCopyInto(out *KubeletAuthorization) { + *out = *in + out.Webhook = in.Webhook + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthorization. +func (in *KubeletAuthorization) DeepCopy() *KubeletAuthorization { + if in == nil { + return nil + } + out := new(KubeletAuthorization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EnableServer != nil { + in, out := &in.EnableServer, &out.EnableServer + *out = new(bool) + **out = **in + } + out.SyncFrequency = in.SyncFrequency + out.FileCheckFrequency = in.FileCheckFrequency + out.HTTPCheckFrequency = in.HTTPCheckFrequency + if in.StaticPodURLHeader != nil { + in, out := &in.StaticPodURLHeader, &out.StaticPodURLHeader + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.TLSCipherSuites != nil { + in, out := &in.TLSCipherSuites, &out.TLSCipherSuites + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Authentication.DeepCopyInto(&out.Authentication) + out.Authorization = in.Authorization + if in.RegistryPullQPS != nil { + in, out := &in.RegistryPullQPS, &out.RegistryPullQPS + *out = new(int32) + **out = **in + } + if in.EventRecordQPS != nil { + in, out := &in.EventRecordQPS, &out.EventRecordQPS + *out = new(int32) + **out = **in + } + if in.EnableDebuggingHandlers != nil { + in, out := &in.EnableDebuggingHandlers, &out.EnableDebuggingHandlers + *out = new(bool) + **out = **in + } + if in.HealthzPort != nil { + in, out := &in.HealthzPort, &out.HealthzPort + *out = new(int32) + **out = **in + } + if in.OOMScoreAdj != nil { + in, out := &in.OOMScoreAdj, &out.OOMScoreAdj + *out = new(int32) + **out = **in + } + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout + out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + out.NodeStatusReportFrequency = in.NodeStatusReportFrequency + out.ImageMinimumGCAge = in.ImageMinimumGCAge + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod + if in.CgroupsPerQOS != nil { + in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS + *out = new(bool) + **out = **in + } + if in.CPUManagerPolicyOptions != nil { + in, out := &in.CPUManagerPolicyOptions, &out.CPUManagerPolicyOptions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod + if in.QOSReserved != nil { + in, out := &in.QOSReserved, &out.QOSReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.RuntimeRequestTimeout = in.RuntimeRequestTimeout + if in.PodPidsLimit != nil { + in, out := &in.PodPidsLimit, &out.PodPidsLimit + *out = new(int64) + **out = **in + } + if in.ResolverConfig != nil { + in, out := &in.ResolverConfig, &out.ResolverConfig + *out = new(string) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } + if in.CPUCFSQuotaPeriod != nil { + in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod + *out = new(v1.Duration) + **out = **in + } + if in.NodeStatusMaxImages != nil { + in, out := &in.NodeStatusMaxImages, &out.NodeStatusMaxImages + *out = new(int32) + **out = **in + } + if in.KubeAPIQPS != nil { + in, out := &in.KubeAPIQPS, &out.KubeAPIQPS + *out = new(int32) + **out = **in + } + if in.SerializeImagePulls != nil { + in, out := &in.SerializeImagePulls, &out.SerializeImagePulls + *out = new(bool) + **out = **in + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod + if in.EvictionMinimumReclaim != nil { + in, out := &in.EvictionMinimumReclaim, &out.EvictionMinimumReclaim + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EnableControllerAttachDetach != nil { + in, out := &in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach + *out = new(bool) + **out = **in + } + if in.MakeIPTablesUtilChains != nil { + in, out := &in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains + *out = new(bool) + **out = **in + } + if in.IPTablesMasqueradeBit != nil { + in, out := &in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit + *out = new(int32) + **out = **in + } + if in.IPTablesDropBit != nil { + in, out := &in.IPTablesDropBit, &out.IPTablesDropBit + *out = new(int32) + **out = **in + } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn + *out = new(bool) + **out = **in + } + out.MemorySwap = in.MemorySwap + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EnforceNodeAllocatable != nil { + in, out := &in.EnforceNodeAllocatable, &out.EnforceNodeAllocatable + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Logging.DeepCopyInto(&out.Logging) + if in.EnableSystemLogHandler != nil { + in, out := &in.EnableSystemLogHandler, &out.EnableSystemLogHandler + *out = new(bool) + **out = **in + } + out.ShutdownGracePeriod = in.ShutdownGracePeriod + out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods + if in.ShutdownGracePeriodByPodPriority != nil { + in, out := &in.ShutdownGracePeriodByPodPriority, &out.ShutdownGracePeriodByPodPriority + *out = make([]ShutdownGracePeriodByPodPriority, len(*in)) + copy(*out, *in) + } + if in.ReservedMemory != nil { + in, out := &in.ReservedMemory, &out.ReservedMemory + *out = make([]MemoryReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.EnableProfilingHandler != nil { + in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler + *out = new(bool) + **out = **in + } + if in.EnableDebugFlagsHandler != nil { + in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler + *out = new(bool) + **out = **in + } + if in.SeccompDefault != nil { + in, out := &in.SeccompDefault, &out.SeccompDefault + *out = new(bool) + **out = **in + } + if in.MemoryThrottlingFactor != nil { + in, out := &in.MemoryThrottlingFactor, &out.MemoryThrottlingFactor + *out = new(float64) + **out = **in + } + if in.RegisterWithTaints != nil { + in, out := &in.RegisterWithTaints, &out.RegisterWithTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RegisterNode != nil { + in, out := &in.RegisterNode, &out.RegisterNode + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubeletConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletWebhookAuthentication) DeepCopyInto(out *KubeletWebhookAuthentication) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + out.CacheTTL = in.CacheTTL + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthentication. +func (in *KubeletWebhookAuthentication) DeepCopy() *KubeletWebhookAuthentication { + if in == nil { + return nil + } + out := new(KubeletWebhookAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletWebhookAuthorization) DeepCopyInto(out *KubeletWebhookAuthorization) { + *out = *in + out.CacheAuthorizedTTL = in.CacheAuthorizedTTL + out.CacheUnauthorizedTTL = in.CacheUnauthorizedTTL + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthorization. +func (in *KubeletWebhookAuthorization) DeepCopy() *KubeletWebhookAuthorization { + if in == nil { + return nil + } + out := new(KubeletWebhookAuthorization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletX509Authentication) DeepCopyInto(out *KubeletX509Authentication) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletX509Authentication. +func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { + if in == nil { + return nil + } + out := new(KubeletX509Authentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryReservation) DeepCopyInto(out *MemoryReservation) { + *out = *in + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryReservation. +func (in *MemoryReservation) DeepCopy() *MemoryReservation { + if in == nil { + return nil + } + out := new(MemoryReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemorySwapConfiguration) DeepCopyInto(out *MemorySwapConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemorySwapConfiguration. +func (in *MemorySwapConfiguration) DeepCopy() *MemorySwapConfiguration { + if in == nil { + return nil + } + out := new(MemorySwapConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerializedNodeConfigSource. +func (in *SerializedNodeConfigSource) DeepCopy() *SerializedNodeConfigSource { + if in == nil { + return nil + } + out := new(SerializedNodeConfigSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SerializedNodeConfigSource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShutdownGracePeriodByPodPriority) DeepCopyInto(out *ShutdownGracePeriodByPodPriority) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShutdownGracePeriodByPodPriority. +func (in *ShutdownGracePeriodByPodPriority) DeepCopy() *ShutdownGracePeriodByPodPriority { + if in == nil { + return nil + } + out := new(ShutdownGracePeriodByPodPriority) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a790204201..a1879415f1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1517,6 +1517,9 @@ k8s.io/kubectl/pkg/util/openapi/validation k8s.io/kubectl/pkg/util/templates k8s.io/kubectl/pkg/util/term k8s.io/kubectl/pkg/validation +# k8s.io/kubelet v0.23.1 => k8s.io/kubelet v0.23.1 +## explicit; go 1.16 +k8s.io/kubelet/config/v1beta1 # k8s.io/legacy-cloud-providers v0.23.1 => k8s.io/legacy-cloud-providers v0.23.1 ## explicit; go 1.16 k8s.io/legacy-cloud-providers/aws