containerd: introduce a new field containerd.nri to enable NRI

Node Resource Interface (NRI) is a common framework for plugging
domain or vendor-specific custom logic into container runtime like
containerd. This commit introduces a new congiguration field
`containerd.nri`, providing cluster admins the flexibility to opt
in for this feature in containerd and tune some of its parameters.
By default, NRI is disabled here in accordance with the containerd's
default config file.

Signed-off-by: Feruzjon Muyassarov <feruzjon.muyassarov@intel.com>
This commit is contained in:
Feruzjon Muyassarov 2023-10-05 00:29:23 +03:00
parent ed6edaf193
commit 0aeab5e523
14 changed files with 403 additions and 0 deletions

View File

@ -1300,6 +1300,25 @@ spec:
- http://HostIP2:Port2
```
### NRI configuration
Using kOps, you can activate the [Node Resource Interface](https://github.com/containerd/nri) (NRI) feature in containerd. It's important to have a at least containerd version of [1.7.0](https://github.com/containerd/containerd/releases/tag/v1.7.0) or later. The available NRI parameters for containerd in kOps include: `enabled`, `pluginRegistrationTimeout` and `pluginRequestTimeout`. By default, NRI options are unset in kOps, which means we rely on containerd's default behavior (i.e., disabled).
```yaml
spec:
containerd:
version: 1.7.0
nri:
# Enable NRI support in containerd.
enabled: true
# pluginRegistrationTimeout is the timeout for a plugin to register after connection.
pluginRegistrationTimeout: "5s"
# pluginRequestTimeout is the timeout for a plugin to handle an event/request.
pluginRequestTimeout: "2s"
```
If you have NRI disabled (i.e., `nri.enabled = false`), please note that settings for `pluginRegistrationTimeout`, and `pluginRequestTimeout` won't take effect. These settings are only applicable when NRI is enabled. It is valid configuration to enable NRI without specifying custom values for `pluginRegistrationTimeout`, and `pluginRequestTimeout`, as these fields will inherit their default values from containerd. If you need to configure additional NRI parameters, you can do so by providing your complete containerd configuration using `configOverride`.
## Docker
It is possible to override Docker daemon options for all masters and nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#DockerConfig) for the full list of options.

View File

@ -821,6 +821,21 @@ spec:
description: LogLevel controls the logging details [trace, debug,
info, warn, error, fatal, panic] (default "info").
type: string
nri:
description: NRI configures the Node Resource Interface.
properties:
enabled:
description: Enable NRI support in containerd
type: boolean
pluginRegistrationTimeout:
description: PluginRegistrationTimeout is the timeout for
plugin registration
type: string
pluginRequestTimeout:
description: PluginRequestTimeout is the timeout for a plugin
to handle a request
type: string
type: object
nvidiaGPU:
description: NvidiaGPU configures the Nvidia GPU runtime.
properties:

View File

@ -125,6 +125,21 @@ spec:
description: LogLevel controls the logging details [trace, debug,
info, warn, error, fatal, panic] (default "info").
type: string
nri:
description: NRI configures the Node Resource Interface.
properties:
enabled:
description: Enable NRI support in containerd
type: boolean
pluginRegistrationTimeout:
description: PluginRegistrationTimeout is the timeout for
plugin registration
type: string
pluginRequestTimeout:
description: PluginRequestTimeout is the timeout for a plugin
to handle a request
type: string
type: object
nvidiaGPU:
description: NvidiaGPU configures the Nvidia GPU runtime.
properties:

View File

@ -479,6 +479,16 @@ func (b *ContainerdBuilder) buildContainerdConfig() (string, error) {
config, _ := toml.Load("")
config.SetPath([]string{"version"}, int64(2))
if containerd.NRI != nil && (containerd.NRI.Enabled == nil || fi.ValueOf(containerd.NRI.Enabled)) {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "disable"}, false)
if containerd.NRI.PluginRequestTimeout != nil {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "plugin_request_timeout"}, containerd.NRI.PluginRequestTimeout)
}
if containerd.NRI.PluginRegistrationTimeout != nil {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "plugin_registration_timeout"}, containerd.NRI.PluginRegistrationTimeout)
}
}
if containerd.SeLinuxEnabled {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_selinux"}, true)
}

View File

@ -16,6 +16,10 @@ limitations under the License.
package kops
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NvidiaDefaultDriverPackage is the nvidia driver default version
const NvidiaDefaultDriverPackage = "nvidia-headless-515-server"
@ -45,6 +49,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -16,6 +16,8 @@ limitations under the License.
package v1alpha2
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// ContainerdConfig is the configuration for containerd
type ContainerdConfig struct {
// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
@ -42,6 +44,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -814,6 +814,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NRIConfig)(nil), (*kops.NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(a.(*NRIConfig), b.(*kops.NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*kops.NRIConfig)(nil), (*NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(a.(*kops.NRIConfig), b.(*NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NTPConfig)(nil), (*kops.NTPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha2_NTPConfig_To_kops_NTPConfig(a.(*NTPConfig), b.(*kops.NTPConfig), scope)
}); err != nil {
@ -3177,6 +3187,15 @@ func autoConvert_v1alpha2_ContainerdConfig_To_kops_ContainerdConfig(in *Containe
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(kops.NRIConfig)
if err := Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -3222,6 +3241,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha2_ContainerdConfig(in *kops.Con
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
if err := Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -5920,6 +5948,30 @@ func Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(
return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in, out, s)
}
func autoConvert_v1alpha2_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_v1alpha2_NRIConfig_To_kops_NRIConfig is an autogenerated conversion function.
func Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
return autoConvert_v1alpha2_NRIConfig_To_kops_NRIConfig(in, out, s)
}
func autoConvert_kops_NRIConfig_To_v1alpha2_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_kops_NRIConfig_To_v1alpha2_NRIConfig is an autogenerated conversion function.
func Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
return autoConvert_kops_NRIConfig_To_v1alpha2_NRIConfig(in, out, s)
}
func autoConvert_v1alpha2_NTPConfig_To_kops_NTPConfig(in *NTPConfig, out *kops.NTPConfig, s conversion.Scope) error {
out.Managed = in.Managed
return nil

View File

@ -1472,6 +1472,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4399,6 +4404,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in

View File

@ -16,6 +16,8 @@ limitations under the License.
package v1alpha3
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// ContainerdConfig is the configuration for containerd
type ContainerdConfig struct {
// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
@ -42,6 +44,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -924,6 +924,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NRIConfig)(nil), (*kops.NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(a.(*NRIConfig), b.(*kops.NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*kops.NRIConfig)(nil), (*NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(a.(*kops.NRIConfig), b.(*NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NTPConfig)(nil), (*kops.NTPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha3_NTPConfig_To_kops_NTPConfig(a.(*NTPConfig), b.(*kops.NTPConfig), scope)
}); err != nil {
@ -3422,6 +3432,15 @@ func autoConvert_v1alpha3_ContainerdConfig_To_kops_ContainerdConfig(in *Containe
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(kops.NRIConfig)
if err := Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -3467,6 +3486,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha3_ContainerdConfig(in *kops.Con
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
if err := Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -6291,6 +6319,30 @@ func Convert_kops_MixedInstancesPolicySpec_To_v1alpha3_MixedInstancesPolicySpec(
return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha3_MixedInstancesPolicySpec(in, out, s)
}
func autoConvert_v1alpha3_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_v1alpha3_NRIConfig_To_kops_NRIConfig is an autogenerated conversion function.
func Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
return autoConvert_v1alpha3_NRIConfig_To_kops_NRIConfig(in, out, s)
}
func autoConvert_kops_NRIConfig_To_v1alpha3_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_kops_NRIConfig_To_v1alpha3_NRIConfig is an autogenerated conversion function.
func Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
return autoConvert_kops_NRIConfig_To_v1alpha3_NRIConfig(in, out, s)
}
func autoConvert_v1alpha3_NTPConfig_To_kops_NTPConfig(in *NTPConfig, out *kops.NTPConfig, s conversion.Scope) error {
out.Managed = in.Managed
return nil

View File

@ -1378,6 +1378,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4345,6 +4350,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in

View File

@ -1686,6 +1686,10 @@ func validateContainerdConfig(spec *kops.ClusterSpec, config *kops.ContainerdCon
}
}
if config.NRI != nil {
allErrs = append(allErrs, validateNriConfig(config, fldPath.Child("nri"))...)
}
if config.Packages != nil {
if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil {
u := fi.ValueOf(config.Packages.UrlAmd64)
@ -1735,6 +1739,26 @@ func validateContainerdConfig(spec *kops.ClusterSpec, config *kops.ContainerdCon
return allErrs
}
func validateNriConfig(containerd *kops.ContainerdConfig, fldPath *field.Path) (allErrs field.ErrorList) {
if containerd.NRI.Enabled == nil || !fi.ValueOf(containerd.NRI.Enabled) {
return allErrs
}
v, err := semver.Parse(*containerd.Version)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), containerd.Version,
fmt.Sprintf("unable to parse version string: %s", err.Error())))
}
expectedRange, err := semver.ParseRange(">=1.7.0")
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), containerd.Version,
fmt.Sprintf("unable to parse version range: %s", err.Error())))
}
if !expectedRange(v) {
allErrs = append(allErrs, field.Forbidden(fldPath, "NRI is available starting from version 1.7.0 and above"))
}
return allErrs
}
func validateNvidiaConfig(spec *kops.ClusterSpec, nvidia *kops.NvidiaGPUConfig, fldPath *field.Path, inClusterConfig bool) (allErrs field.ErrorList) {
if !fi.ValueOf(nvidia.Enabled) {
return allErrs

View File

@ -1532,3 +1532,70 @@ func Test_Validate_Nvidia_Ig(t *testing.T) {
testErrors(t, g.Input, errs, g.ExpectedErrors)
}
}
func Test_Validate_NriConfig(t *testing.T) {
unsupportedContainerdVersion := "1.6.0"
supportedContainerdVersion := "1.7.0"
grid := []struct {
Input kops.ClusterSpec
ExpectedErrors []string
}{
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(true),
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{"Forbidden::containerd.nri"},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: nil,
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(false),
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(true),
},
Version: &supportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
}
for _, g := range grid {
errs := validateNriConfig(g.Input.Containerd, field.NewPath("containerd", "nri"))
testErrors(t, g.Input.Containerd, errs, g.ExpectedErrors)
}
}

View File

@ -1498,6 +1498,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4563,6 +4568,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in