kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13128 from hakman/kubelet_dockershim_cleanup 

Clean up kubelet networking flags for dockershim
This commit is contained in:
Kubernetes Prow Robot 2022-01-19 10:14:06 -08:00 committed by GitHub
parent 8e8ae77a15 68b4611066
commit 433cca7471
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 87 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
nodeup/pkg/model/kubelet.go
View File

@ -259,11 +259,16 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
// Add container runtime spcific flags
switch b.Cluster.Spec.ContainerRuntime {
case "docker", "":
flags += " --cni-bin-dir=" + b.CNIBinDir()
flags += " --cni-conf-dir=" + b.CNIConfDir()
case "docker":
if b.IsKubernetesLT("1.24") {
flags += " --container-runtime=docker"
flags += " --cni-bin-dir=" + b.CNIBinDir()
flags += " --cni-conf-dir=" + b.CNIConfDir()
}
case "containerd":
flags += " --container-runtime=remote"
if b.IsKubernetesLT("1.24") {
flags += " --container-runtime=remote"
}
flags += " --runtime-request-timeout=15m"
if b.Cluster.Spec.Containerd == nil || b.Cluster.Spec.Containerd.Address == nil {
flags += " --container-runtime-endpoint=unix:///run/containerd/containerd.sock"

4
nodeup/pkg/model/tests/kubelet/featuregates/cluster.yaml
View File

@ -9,7 +9,7 @@ spec:
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal.example.com
containerRuntime: docker
containerRuntime: containerd
etcdClusters:
- etcdMembers:
- instanceGroup: master-us-test-1a
@ -25,7 +25,7 @@ spec:
ExperimentalCriticalPodAnnotation: "true"
AllowExtTrafficLocalEndpoints: "false"
podManifestPath: "/etc/kubernetes/manifests"
kubernetesVersion: v1.22.0
kubernetesVersion: v1.24.0
masterInternalName: api.internal.minimal.example.com
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16

4
nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml
View File

@ -3,7 +3,7 @@ path: /etc/kubernetes/manifests
type: directory
---
contents: |
DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=aws --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=AllowExtTrafficLocalEndpoints=false,CSIMigrationAWS=true,ExperimentalCriticalPodAnnotation=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --network-plugin-mtu=9001 --network-plugin=kubenet --non-masquerade-cidr=100.64.0.0/10 --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --cni-bin-dir=/opt/cni/bin/ --cni-conf-dir=/etc/cni/net.d/ --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key"
DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=AllowExtTrafficLocalEndpoints=false,CSIMigrationAWS=true,ExperimentalCriticalPodAnnotation=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key"
HOME="/root"
path: /etc/sysconfig/kubelet
type: file
@ -13,7 +13,7 @@ definition: |
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
After=containerd.service
[Service]
EnvironmentFile=/etc/sysconfig/kubelet

4
nodeup/pkg/model/tests/kubelet/warmpool/cluster.yaml
View File

@ -9,7 +9,7 @@ spec:
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal.example.com
containerRuntime: docker
containerRuntime: containerd
etcdClusters:
- etcdMembers:
- instanceGroup: master-us-test-1a
@ -22,7 +22,7 @@ spec:
iam: {}
kubelet:
podManifestPath: "/etc/kubernetes/manifests"
kubernetesVersion: v1.20.0
kubernetesVersion: v1.24.0
masterInternalName: api.internal.minimal.example.com
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16

4
nodeup/pkg/model/tests/kubelet/warmpool/tasks.yaml
View File

@ -3,7 +3,7 @@ path: /etc/kubernetes/manifests
type: directory
---
contents: |
DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=aws --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --kubeconfig=/var/lib/kubelet/kubeconfig --network-plugin-mtu=9001 --network-plugin=kubenet --non-masquerade-cidr=100.64.0.0/10 --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --cni-bin-dir=/opt/cni/bin/ --cni-conf-dir=/etc/cni/net.d/ --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key"
DAEMON_ARGS="--authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --cgroup-root=/ --client-ca-file=/srv/kubernetes/ca.crt --cloud-provider=external --cluster-dns=100.64.0.10 --cluster-domain=cluster.local --enable-debugging-handlers=true --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true --kubeconfig=/var/lib/kubelet/kubeconfig --pod-infra-container-image=k8s.gcr.io/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --register-schedulable=true --v=2 --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ --cloud-config=/etc/kubernetes/in-tree-cloud.config --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --tls-cert-file=/srv/kubernetes/kubelet-server.crt --tls-private-key-file=/srv/kubernetes/kubelet-server.key"
HOME="/root"
path: /etc/sysconfig/kubelet
type: file
@ -13,7 +13,7 @@ definition: |
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
After=containerd.service
[Service]
EnvironmentFile=/etc/sysconfig/kubelet

4
pkg/apis/kops/componentconfig.go
View File

@ -73,7 +73,7 @@ type KubeletConfigSpec struct {
// ClusterDNS is the IP address for a cluster DNS server
ClusterDNS string `json:"clusterDNS,omitempty" flag:"cluster-dns"`
// NetworkPluginName is the name of the network plugin to be invoked for various events in kubelet/pod lifecycle
NetworkPluginName string `json:"networkPluginName,omitempty" flag:"network-plugin"`
NetworkPluginName *string `json:"networkPluginName,omitempty" flag:"network-plugin"`
// CloudProvider is the provider for cloud services.
CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"`
// KubeletCgroups is the absolute name of cgroups to isolate the kubelet in.
@ -124,7 +124,7 @@ type KubeletConfigSpec struct {
// NodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels,omitempty" flag:"node-labels"`
// NonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
NonMasqueradeCIDR *string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
// Enable gathering custom metrics.
EnableCustomMetrics *bool `json:"enableCustomMetrics,omitempty" flag:"enable-custom-metrics"`
// NetworkPluginMTU is the MTU to be passed to the network plugin,

4
pkg/apis/kops/v1alpha2/componentconfig.go
View File

@ -73,7 +73,7 @@ type KubeletConfigSpec struct {
// ClusterDNS is the IP address for a cluster DNS server
ClusterDNS string `json:"clusterDNS,omitempty" flag:"cluster-dns"`
// NetworkPluginName is the name of the network plugin to be invoked for various events in kubelet/pod lifecycle
NetworkPluginName string `json:"networkPluginName,omitempty" flag:"network-plugin"`
NetworkPluginName *string `json:"networkPluginName,omitempty" flag:"network-plugin"`
// CloudProvider is the provider for cloud services.
CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"`
// KubeletCgroups is the absolute name of cgroups to isolate the kubelet in.
@ -124,7 +124,7 @@ type KubeletConfigSpec struct {
// NodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels,omitempty" flag:"node-labels"`
// NonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
NonMasqueradeCIDR *string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
// Enable gathering custom metrics.
EnableCustomMetrics *bool `json:"enableCustomMetrics,omitempty" flag:"enable-custom-metrics"`
// NetworkPluginMTU is the MTU to be passed to the network plugin,

10
pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go generated
View File

@ -3532,6 +3532,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
*out = new(v1.Duration)
**out = **in
}
if in.NetworkPluginName != nil {
in, out := &in.NetworkPluginName, &out.NetworkPluginName
*out = new(string)
**out = **in
}
if in.ReadOnlyPort != nil {
in, out := &in.ReadOnlyPort, &out.ReadOnlyPort
*out = new(int32)
@ -3579,6 +3584,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
(*out)[key] = val
}
}
if in.NonMasqueradeCIDR != nil {
in, out := &in.NonMasqueradeCIDR, &out.NonMasqueradeCIDR
*out = new(string)
**out = **in
}
if in.EnableCustomMetrics != nil {
in, out := &in.EnableCustomMetrics, &out.EnableCustomMetrics
*out = new(bool)

4
pkg/apis/kops/v1alpha3/componentconfig.go
View File

@ -73,7 +73,7 @@ type KubeletConfigSpec struct {
// ClusterDNS is the IP address for a cluster DNS server
ClusterDNS string `json:"clusterDNS,omitempty" flag:"cluster-dns"`
// NetworkPluginName is the name of the network plugin to be invoked for various events in kubelet/pod lifecycle
NetworkPluginName string `json:"networkPluginName,omitempty" flag:"network-plugin"`
NetworkPluginName *string `json:"networkPluginName,omitempty" flag:"network-plugin"`
// CloudProvider is the provider for cloud services.
CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"`
// KubeletCgroups is the absolute name of cgroups to isolate the kubelet in.
@ -123,7 +123,7 @@ type KubeletConfigSpec struct {
// NodeLabels is not admin-configurable.
NodeLabels map[string]string `json:"-"`
// NonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
NonMasqueradeCIDR *string `json:"nonMasqueradeCIDR,omitempty" flag:"non-masquerade-cidr"`
// EnableCustomMetrics was removed.
EnableCustomMetrics *bool `json:"-"`
// NetworkPluginMTU is the MTU to be passed to the network plugin,

10
pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go generated
View File

@ -3438,6 +3438,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
*out = new(v1.Duration)
**out = **in
}
if in.NetworkPluginName != nil {
in, out := &in.NetworkPluginName, &out.NetworkPluginName
*out = new(string)
**out = **in
}
if in.ReadOnlyPort != nil {
in, out := &in.ReadOnlyPort, &out.ReadOnlyPort
*out = new(int32)
@ -3485,6 +3490,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
(*out)[key] = val
}
}
if in.NonMasqueradeCIDR != nil {
in, out := &in.NonMasqueradeCIDR, &out.NonMasqueradeCIDR
*out = new(string)
**out = **in
}
if in.EnableCustomMetrics != nil {
in, out := &in.EnableCustomMetrics, &out.EnableCustomMetrics
*out = new(bool)

10
pkg/apis/kops/validation/legacy.go
View File

@ -181,14 +181,14 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("nonMasqueradeCIDR"), fmt.Sprintf("nonMasqueradeCIDR %q cannot overlap with networkCIDR %q", nonMasqueradeCIDRString, c.Spec.NetworkCIDR)))
}
if c.Spec.ContainerRuntime == "docker" && c.Spec.Kubelet != nil && c.Spec.Kubelet.NetworkPluginName == "kubenet" {
if c.Spec.Kubelet.NonMasqueradeCIDR != nonMasqueradeCIDRString {
if strict || c.Spec.Kubelet.NonMasqueradeCIDR != "" {
if c.Spec.ContainerRuntime == "docker" && c.Spec.Kubelet != nil && fi.StringValue(c.Spec.Kubelet.NetworkPluginName) == "kubenet" {
if fi.StringValue(c.Spec.Kubelet.NonMasqueradeCIDR) != nonMasqueradeCIDRString {
if strict || c.Spec.Kubelet.NonMasqueradeCIDR != nil {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "nonMasqueradeCIDR"), "kubelet nonMasqueradeCIDR did not match cluster nonMasqueradeCIDR"))
}
}
if c.Spec.MasterKubelet.NonMasqueradeCIDR != nonMasqueradeCIDRString {
if strict || c.Spec.MasterKubelet.NonMasqueradeCIDR != "" {
if fi.StringValue(c.Spec.MasterKubelet.NonMasqueradeCIDR) != nonMasqueradeCIDRString {
if strict || c.Spec.MasterKubelet.NonMasqueradeCIDR != nil {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("masterKubelet", "nonMasqueradeCIDR"), "masterKubelet nonMasqueradeCIDR did not match cluster nonMasqueradeCIDR"))
}
}

21
pkg/apis/kops/validation/validation.go
View File

@ -642,6 +642,15 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi
if k != nil {
{
// Flag removed in 1.5
if k.ConfigureCBR0 != nil {
allErrs = append(allErrs, field.Forbidden(
kubeletPath.Child("ConfigureCBR0"),
"configure-cbr0 flag was removed in 1.5"))
}
}
{
// Flag removed in 1.6
if k.APIServers != "" {
@ -692,6 +701,18 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi
}
}
if c.IsKubernetesGTE("1.24") {
if k.NetworkPluginName != nil {
allErrs = append(allErrs, field.Forbidden(kubeletPath.Child("networkPluginName"), "networkPluginName has been removed on Kubernetes >=1.24"))
}
if k.NetworkPluginMTU != nil {
allErrs = append(allErrs, field.Forbidden(kubeletPath.Child("networkPluginMTU"), "networkPluginMTU has been removed on Kubernetes >=1.24"))
}
if k.NonMasqueradeCIDR != nil {
allErrs = append(allErrs, field.Forbidden(kubeletPath.Child("nonMasqueradeCIDR"), "nonMasqueradeCIDR has been removed on Kubernetes >=1.24"))
}
}
}
return allErrs
}

10
pkg/apis/kops/zz_generated.deepcopy.go generated
View File

@ -3617,6 +3617,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
*out = new(v1.Duration)
**out = **in
}
if in.NetworkPluginName != nil {
in, out := &in.NetworkPluginName, &out.NetworkPluginName
*out = new(string)
**out = **in
}
if in.ReadOnlyPort != nil {
in, out := &in.ReadOnlyPort, &out.ReadOnlyPort
*out = new(int32)
@ -3664,6 +3669,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
(*out)[key] = val
}
}
if in.NonMasqueradeCIDR != nil {
in, out := &in.NonMasqueradeCIDR, &out.NonMasqueradeCIDR
*out = new(string)
**out = **in
}
if in.EnableCustomMetrics != nil {
in, out := &in.EnableCustomMetrics, &out.EnableCustomMetrics
*out = new(bool)

4
pkg/model/components/kubelet.go
View File

@ -163,9 +163,9 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error {
return fmt.Errorf("no networking mode set")
}
if UsesKubenet(networking) && b.IsKubernetesLT("1.24") {
clusterSpec.Kubelet.NetworkPluginName = "kubenet"
clusterSpec.Kubelet.NetworkPluginName = fi.String("kubenet")
clusterSpec.Kubelet.NetworkPluginMTU = fi.Int32(9001)
clusterSpec.Kubelet.NonMasqueradeCIDR = clusterSpec.NonMasqueradeCIDR
clusterSpec.Kubelet.NonMasqueradeCIDR = fi.String(clusterSpec.NonMasqueradeCIDR)
}
}

17
pkg/model/components/networking.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/loader"
)
@ -43,18 +44,12 @@ func (b *NetworkingOptionsBuilder) BuildOptions(o interface{}) error {
return fmt.Errorf("networking not set")
}
if UsesCNI(networking) {
if b.Context.IsKubernetesLT("1.24") {
options.Kubelet.NetworkPluginName = "cni"
if b.Context.IsKubernetesLT("1.24") {
if UsesCNI(networking) {
options.Kubelet.NetworkPluginName = fi.String("cni")
} else if networking.GCE != nil {
options.Kubelet.NetworkPluginName = fi.String("kubenet")
}
// ConfigureCBR0 flag removed from 1.5
options.Kubelet.ConfigureCBR0 = nil
}
if networking.GCE != nil {
// GCE IPAlias networking uses kubenet on the nodes
options.Kubelet.NetworkPluginName = "kubenet"
}
if networking.Classic != nil {

45
upup/pkg/fi/cloudup/populate_cluster_spec_test.go
View File

@ -195,51 +195,6 @@ func build(c *kopsapi.Cluster) (*kopsapi.Cluster, error) {
return full, nil
}
func TestPopulateCluster_Kubenet(t *testing.T) {
_, c := buildMinimalCluster()
full, err := build(c)
if err != nil {
t.Fatalf("error during build: %v", err)
}
if full.Spec.Kubelet.NetworkPluginName != "kubenet" {
t.Fatalf("Unexpected NetworkPluginName: %v", full.Spec.Kubelet.NetworkPluginName)
}
if fi.BoolValue(full.Spec.KubeControllerManager.ConfigureCloudRoutes) != true {
t.Fatalf("Unexpected ConfigureCloudRoutes: %v", full.Spec.KubeControllerManager.ConfigureCloudRoutes)
}
}
func TestPopulateCluster_CNI(t *testing.T) {
_, c := buildMinimalCluster()
c.Spec.Kubelet = &kopsapi.KubeletConfigSpec{
ConfigureCBR0: fi.Bool(false),
NetworkPluginName: "cni",
NonMasqueradeCIDR: c.Spec.NonMasqueradeCIDR,
CloudProvider: c.Spec.CloudProvider,
}
full, err := build(c)
if err != nil {
t.Fatalf("error during build: %v", err)
}
if full.Spec.Kubelet.NetworkPluginName != "cni" {
t.Fatalf("Unexpected NetworkPluginName: %v", full.Spec.Kubelet.NetworkPluginName)
}
if fi.BoolValue(full.Spec.Kubelet.ConfigureCBR0) != false {
t.Fatalf("Unexpected ConfigureCBR0: %v", full.Spec.Kubelet.ConfigureCBR0)
}
if fi.BoolValue(full.Spec.KubeControllerManager.ConfigureCloudRoutes) != true {
t.Fatalf("Unexpected ConfigureCloudRoutes: %v", full.Spec.KubeControllerManager.ConfigureCloudRoutes)
}
}
func TestPopulateCluster_Custom_CIDR(t *testing.T) {
cloud, c := buildMinimalCluster()
c.Spec.NetworkCIDR = "172.20.2.0/24"