Merge pull request #7730 from ripta/custom-sysctls

Custom sysctl Parameters

docs/cluster_spec.md

@@ -870,3 +870,28 @@ spec:
   assets:
     containerProxy: proxy.example.com
 ```
+
+### Setting Custom Kernel Runtime Parameters
+
+To add custom kernel runtime parameters to your all instance groups in the
+cluster, specify the `sysctlParameters` field as an array of strings. Each
+string must take the form of `variable=value` the way it would appear in
+sysctl.conf (see also `sysctl(8)` manpage).
+
+You could also use the `sysctlParameters` field on [the instance group](https://github.com/kubernetes/kops/blob/master/docs/instance_groups.md#setting-custom-kernel-runtime-parameters) to specify different parameters for each instance group.
+
+Unlike a simple file asset, specifying kernel runtime parameters in this manner
+would correctly invoke `sysctl --system` automatically for you to apply said
+parameters.
+
+For example:
+
+```yaml
+spec:
+  sysctlParameters:
+    - fs.pipe-user-pages-soft=524288
+    - net.ipv4.tcp_keepalive_time=200
+```
+
+which would end up in a drop-in file on all masters and nodes of the cluster.
+

docs/instance_groups.md

@@ -574,3 +574,31 @@ spec:
 ```
 
 If `openstack.kops.io/osVolumeSize` is not set it will default to the minimum disk specified by the image.
+
+
+## Setting Custom Kernel Runtime Parameters
+
+To add custom kernel runtime parameters to your instance group, specify the
+`sysctlParameters` field as an array of strings. Each string must take the form
+of `variable=value` the way it would appear in sysctl.conf (see also
+`sysctl(8)` manpage).
+
+Unlike a simple file asset, specifying kernel runtime parameters in this manner
+would correctly invoke `sysctl --system` automatically for you to apply said
+parameters.
+
+For example:
+
+```yaml
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: nodes
+spec:
+  sysctlParameters:
+    - fs.pipe-user-pages-soft=524288
+    - net.ipv4.tcp_keepalive_time=200
+```
+
+which would end up in a drop-in file on nodes of the instance group in question.
+

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2916,6 +2916,13 @@ spec:
                     type: string
                 type: object
               type: array
+            sysctlParameters:
+              description: SysctlParameters will configure kernel parameters using
+                sysctl(8). When specified, each parameter must follow the form variable=value,
+                the way it would appear in sysctl.conf.
+              items:
+                type: string
+              type: array
             target:
               description: Target allows for us to nest extra config for targets such
                 as terraform

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -664,6 +664,13 @@ spec:
               items:
                 type: string
               type: array
+            sysctlParameters:
+              description: SysctlParameters will configure kernel parameters using
+                sysctl(8). When specified, each parameter must follow the form variable=value,
+                the way it would appear in sysctl.conf.
+              items:
+                type: string
+              type: array
             taints:
               description: Taints indicates the kubernetes taints for nodes in this
                 group

nodeup/pkg/model/sysctls.go

@@ -17,6 +17,7 @@ limitations under the License.
 package model
 
 import (
+	"fmt"
 	"strings"
 
 	"k8s.io/kops/pkg/apis/kops"
@@ -130,6 +131,30 @@ func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
 		"net.ipv4.ip_forward=1",
 		"")
 
+	if params := b.InstanceGroup.Spec.SysctlParameters; len(params) > 0 {
+		sysctls = append(sysctls,
+			"# Custom sysctl parameters from instance group spec",
+			"")
+		for _, param := range params {
+			if !strings.ContainsRune(param, '=') {
+				return fmt.Errorf("Invalid SysctlParameter: expected %q to contain '='", param)
+			}
+			sysctls = append(sysctls, param)
+		}
+	}
+
+	if params := b.Cluster.Spec.SysctlParameters; len(params) > 0 {
+		sysctls = append(sysctls,
+			"# Custom sysctl parameters from cluster spec",
+			"")
+		for _, param := range params {
+			if !strings.ContainsRune(param, '=') {
+				return fmt.Errorf("Invalid SysctlParameter: expected %q to contain '='", param)
+			}
+			sysctls = append(sysctls, param)
+		}
+	}
+
 	c.AddTask(&nodetasks.File{
 		Path:            "/etc/sysctl.d/99-k8s-general.conf",
 		Contents:        fi.NewStringResource(strings.Join(sysctls, "\n")),

pkg/apis/kops/cluster.go

@@ -182,6 +182,10 @@ type ClusterSpec struct {
 	// UseHostCertificates will mount /etc/ssl/certs to inside needed containers.
 	// This is needed if some APIs do have self-signed certs
 	UseHostCertificates *bool `json:"useHostCertificates,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/instancegroup.go

@@ -155,6 +155,10 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 	// InstanceProtection makes new instances in an autoscaling group protected from scale in
 	InstanceProtection *bool `json:"instanceProtection,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha1/cluster.go

@@ -180,6 +180,10 @@ type ClusterSpec struct {
 	// UseHostCertificates will mount /etc/ssl/certs to inside needed containers.
 	// This is needed if some APIs do have self-signed certs
 	UseHostCertificates *bool `json:"useHostCertificates,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/v1alpha1/instancegroup.go

@@ -142,6 +142,10 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 	// InstanceProtection makes new instances in an autoscaling group protected from scale in
 	InstanceProtection *bool `json:"instanceProtection,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1877,6 +1877,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 		out.Target = nil
 	}
 	out.UseHostCertificates = in.UseHostCertificates
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -2174,6 +2175,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 		out.Target = nil
 	}
 	out.UseHostCertificates = in.UseHostCertificates
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -3055,6 +3057,7 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	}
 	out.SecurityGroupOverride = in.SecurityGroupOverride
 	out.InstanceProtection = in.InstanceProtection
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -3177,6 +3180,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
 	}
 	out.SecurityGroupOverride = in.SecurityGroupOverride
 	out.InstanceProtection = in.InstanceProtection
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -808,6 +808,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -1723,6 +1728,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/v1alpha2/cluster.go

@@ -180,6 +180,10 @@ type ClusterSpec struct {
 	// UseHostCertificates will mount /etc/ssl/certs to inside needed containers.
 	// This is needed if some APIs do have self-signed certs
 	UseHostCertificates *bool `json:"useHostCertificates,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -149,6 +149,10 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 	// InstanceProtection makes new instances in an autoscaling group protected from scale in
 	InstanceProtection *bool `json:"instanceProtection,omitempty"`
+	// SysctlParameters will configure kernel parameters using sysctl(8). When
+	// specified, each parameter must follow the form variable=value, the way
+	// it would appear in sysctl.conf.
+	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1930,6 +1930,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 		out.Target = nil
 	}
 	out.UseHostCertificates = in.UseHostCertificates
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -2242,6 +2243,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 		out.Target = nil
 	}
 	out.UseHostCertificates = in.UseHostCertificates
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -3173,6 +3175,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	}
 	out.SecurityGroupOverride = in.SecurityGroupOverride
 	out.InstanceProtection = in.InstanceProtection
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 
@@ -3300,6 +3303,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 	}
 	out.SecurityGroupOverride = in.SecurityGroupOverride
 	out.InstanceProtection = in.InstanceProtection
+	out.SysctlParameters = in.SysctlParameters
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -781,6 +781,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -1685,6 +1690,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -881,6 +881,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -1851,6 +1856,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.SysctlParameters != nil {
+		in, out := &in.SysctlParameters, &out.SysctlParameters
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }