Merge pull request #15994 from fmuyassarov/add-nri-support

containerd: introduce a new field to enable NRI
This commit is contained in:
Kubernetes Prow Robot 2023-10-21 04:52:02 +02:00 committed by GitHub
commit f7bd516b79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 403 additions and 0 deletions

View File

@ -1300,6 +1300,25 @@ spec:
- http://HostIP2:Port2
```
### NRI configuration
Using kOps, you can activate the [Node Resource Interface](https://github.com/containerd/nri) (NRI) feature in containerd. It's important to have a at least containerd version of [1.7.0](https://github.com/containerd/containerd/releases/tag/v1.7.0) or later. The available NRI parameters for containerd in kOps include: `enabled`, `pluginRegistrationTimeout` and `pluginRequestTimeout`. By default, NRI options are unset in kOps, which means we rely on containerd's default behavior (i.e., disabled).
```yaml
spec:
containerd:
version: 1.7.0
nri:
# Enable NRI support in containerd.
enabled: true
# pluginRegistrationTimeout is the timeout for a plugin to register after connection.
pluginRegistrationTimeout: "5s"
# pluginRequestTimeout is the timeout for a plugin to handle an event/request.
pluginRequestTimeout: "2s"
```
If you have NRI disabled (i.e., `nri.enabled = false`), please note that settings for `pluginRegistrationTimeout`, and `pluginRequestTimeout` won't take effect. These settings are only applicable when NRI is enabled. It is valid configuration to enable NRI without specifying custom values for `pluginRegistrationTimeout`, and `pluginRequestTimeout`, as these fields will inherit their default values from containerd. If you need to configure additional NRI parameters, you can do so by providing your complete containerd configuration using `configOverride`.
## Docker
It is possible to override Docker daemon options for all masters and nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#DockerConfig) for the full list of options.

View File

@ -821,6 +821,21 @@ spec:
description: LogLevel controls the logging details [trace, debug,
info, warn, error, fatal, panic] (default "info").
type: string
nri:
description: NRI configures the Node Resource Interface.
properties:
enabled:
description: Enable NRI support in containerd
type: boolean
pluginRegistrationTimeout:
description: PluginRegistrationTimeout is the timeout for
plugin registration
type: string
pluginRequestTimeout:
description: PluginRequestTimeout is the timeout for a plugin
to handle a request
type: string
type: object
nvidiaGPU:
description: NvidiaGPU configures the Nvidia GPU runtime.
properties:

View File

@ -125,6 +125,21 @@ spec:
description: LogLevel controls the logging details [trace, debug,
info, warn, error, fatal, panic] (default "info").
type: string
nri:
description: NRI configures the Node Resource Interface.
properties:
enabled:
description: Enable NRI support in containerd
type: boolean
pluginRegistrationTimeout:
description: PluginRegistrationTimeout is the timeout for
plugin registration
type: string
pluginRequestTimeout:
description: PluginRequestTimeout is the timeout for a plugin
to handle a request
type: string
type: object
nvidiaGPU:
description: NvidiaGPU configures the Nvidia GPU runtime.
properties:

View File

@ -479,6 +479,16 @@ func (b *ContainerdBuilder) buildContainerdConfig() (string, error) {
config, _ := toml.Load("")
config.SetPath([]string{"version"}, int64(2))
if containerd.NRI != nil && (containerd.NRI.Enabled == nil || fi.ValueOf(containerd.NRI.Enabled)) {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "disable"}, false)
if containerd.NRI.PluginRequestTimeout != nil {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "plugin_request_timeout"}, containerd.NRI.PluginRequestTimeout)
}
if containerd.NRI.PluginRegistrationTimeout != nil {
config.SetPath([]string{"plugins", "io.containerd.nri.v1.nri", "plugin_registration_timeout"}, containerd.NRI.PluginRegistrationTimeout)
}
}
if containerd.SeLinuxEnabled {
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "enable_selinux"}, true)
}

View File

@ -16,6 +16,10 @@ limitations under the License.
package kops
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NvidiaDefaultDriverPackage is the nvidia driver default version
const NvidiaDefaultDriverPackage = "nvidia-headless-515-server"
@ -45,6 +49,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -16,6 +16,8 @@ limitations under the License.
package v1alpha2
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// ContainerdConfig is the configuration for containerd
type ContainerdConfig struct {
// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
@ -42,6 +44,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -814,6 +814,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NRIConfig)(nil), (*kops.NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(a.(*NRIConfig), b.(*kops.NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*kops.NRIConfig)(nil), (*NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(a.(*kops.NRIConfig), b.(*NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NTPConfig)(nil), (*kops.NTPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha2_NTPConfig_To_kops_NTPConfig(a.(*NTPConfig), b.(*kops.NTPConfig), scope)
}); err != nil {
@ -3179,6 +3189,15 @@ func autoConvert_v1alpha2_ContainerdConfig_To_kops_ContainerdConfig(in *Containe
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(kops.NRIConfig)
if err := Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -3224,6 +3243,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha2_ContainerdConfig(in *kops.Con
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
if err := Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -5922,6 +5950,30 @@ func Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(
return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in, out, s)
}
func autoConvert_v1alpha2_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_v1alpha2_NRIConfig_To_kops_NRIConfig is an autogenerated conversion function.
func Convert_v1alpha2_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
return autoConvert_v1alpha2_NRIConfig_To_kops_NRIConfig(in, out, s)
}
func autoConvert_kops_NRIConfig_To_v1alpha2_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_kops_NRIConfig_To_v1alpha2_NRIConfig is an autogenerated conversion function.
func Convert_kops_NRIConfig_To_v1alpha2_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
return autoConvert_kops_NRIConfig_To_v1alpha2_NRIConfig(in, out, s)
}
func autoConvert_v1alpha2_NTPConfig_To_kops_NTPConfig(in *NTPConfig, out *kops.NTPConfig, s conversion.Scope) error {
out.Managed = in.Managed
return nil

View File

@ -1477,6 +1477,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4404,6 +4409,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in

View File

@ -16,6 +16,8 @@ limitations under the License.
package v1alpha3
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// ContainerdConfig is the configuration for containerd
type ContainerdConfig struct {
// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
@ -42,6 +44,17 @@ type ContainerdConfig struct {
Runc *Runc `json:"runc,omitempty"`
// SelinuxEnabled enables SELinux support
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
// NRI configures the Node Resource Interface.
NRI *NRIConfig `json:"nri,omitempty"`
}
type NRIConfig struct {
// Enable NRI support in containerd
Enabled *bool `json:"enabled,omitempty"`
// PluginRegistrationTimeout is the timeout for plugin registration
PluginRegistrationTimeout *metav1.Duration `json:"pluginRegistrationTimeout,omitempty"`
// PluginRequestTimeout is the timeout for a plugin to handle a request
PluginRequestTimeout *metav1.Duration `json:"pluginRequestTimeout,omitempty"`
}
type NvidiaGPUConfig struct {

View File

@ -924,6 +924,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NRIConfig)(nil), (*kops.NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(a.(*NRIConfig), b.(*kops.NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*kops.NRIConfig)(nil), (*NRIConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(a.(*kops.NRIConfig), b.(*NRIConfig), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*NTPConfig)(nil), (*kops.NTPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1alpha3_NTPConfig_To_kops_NTPConfig(a.(*NTPConfig), b.(*kops.NTPConfig), scope)
}); err != nil {
@ -3424,6 +3434,15 @@ func autoConvert_v1alpha3_ContainerdConfig_To_kops_ContainerdConfig(in *Containe
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(kops.NRIConfig)
if err := Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -3469,6 +3488,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha3_ContainerdConfig(in *kops.Con
out.Runc = nil
}
out.SeLinuxEnabled = in.SeLinuxEnabled
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
if err := Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(*in, *out, s); err != nil {
return err
}
} else {
out.NRI = nil
}
return nil
}
@ -6295,6 +6323,30 @@ func Convert_kops_MixedInstancesPolicySpec_To_v1alpha3_MixedInstancesPolicySpec(
return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha3_MixedInstancesPolicySpec(in, out, s)
}
func autoConvert_v1alpha3_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_v1alpha3_NRIConfig_To_kops_NRIConfig is an autogenerated conversion function.
func Convert_v1alpha3_NRIConfig_To_kops_NRIConfig(in *NRIConfig, out *kops.NRIConfig, s conversion.Scope) error {
return autoConvert_v1alpha3_NRIConfig_To_kops_NRIConfig(in, out, s)
}
func autoConvert_kops_NRIConfig_To_v1alpha3_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
out.Enabled = in.Enabled
out.PluginRegistrationTimeout = in.PluginRegistrationTimeout
out.PluginRequestTimeout = in.PluginRequestTimeout
return nil
}
// Convert_kops_NRIConfig_To_v1alpha3_NRIConfig is an autogenerated conversion function.
func Convert_kops_NRIConfig_To_v1alpha3_NRIConfig(in *kops.NRIConfig, out *NRIConfig, s conversion.Scope) error {
return autoConvert_kops_NRIConfig_To_v1alpha3_NRIConfig(in, out, s)
}
func autoConvert_v1alpha3_NTPConfig_To_kops_NTPConfig(in *NTPConfig, out *kops.NTPConfig, s conversion.Scope) error {
out.Managed = in.Managed
return nil

View File

@ -1383,6 +1383,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4355,6 +4360,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in

View File

@ -1686,6 +1686,10 @@ func validateContainerdConfig(spec *kops.ClusterSpec, config *kops.ContainerdCon
}
}
if config.NRI != nil {
allErrs = append(allErrs, validateNriConfig(config, fldPath.Child("nri"))...)
}
if config.Packages != nil {
if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil {
u := fi.ValueOf(config.Packages.UrlAmd64)
@ -1735,6 +1739,26 @@ func validateContainerdConfig(spec *kops.ClusterSpec, config *kops.ContainerdCon
return allErrs
}
func validateNriConfig(containerd *kops.ContainerdConfig, fldPath *field.Path) (allErrs field.ErrorList) {
if containerd.NRI.Enabled == nil || !fi.ValueOf(containerd.NRI.Enabled) {
return allErrs
}
v, err := semver.Parse(*containerd.Version)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), containerd.Version,
fmt.Sprintf("unable to parse version string: %s", err.Error())))
}
expectedRange, err := semver.ParseRange(">=1.7.0")
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), containerd.Version,
fmt.Sprintf("unable to parse version range: %s", err.Error())))
}
if !expectedRange(v) {
allErrs = append(allErrs, field.Forbidden(fldPath, "NRI is available starting from version 1.7.0 and above"))
}
return allErrs
}
func validateNvidiaConfig(spec *kops.ClusterSpec, nvidia *kops.NvidiaGPUConfig, fldPath *field.Path, inClusterConfig bool) (allErrs field.ErrorList) {
if !fi.ValueOf(nvidia.Enabled) {
return allErrs

View File

@ -1532,3 +1532,70 @@ func Test_Validate_Nvidia_Ig(t *testing.T) {
testErrors(t, g.Input, errs, g.ExpectedErrors)
}
}
func Test_Validate_NriConfig(t *testing.T) {
unsupportedContainerdVersion := "1.6.0"
supportedContainerdVersion := "1.7.0"
grid := []struct {
Input kops.ClusterSpec
ExpectedErrors []string
}{
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(true),
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{"Forbidden::containerd.nri"},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: nil,
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(false),
},
Version: &unsupportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
{
Input: kops.ClusterSpec{
Containerd: &kops.ContainerdConfig{
NRI: &kops.NRIConfig{
Enabled: fi.PtrTo(true),
},
Version: &supportedContainerdVersion,
},
},
ExpectedErrors: []string{},
},
}
for _, g := range grid {
errs := validateNriConfig(g.Input.Containerd, field.NewPath("containerd", "nri"))
testErrors(t, g.Input.Containerd, errs, g.ExpectedErrors)
}
}

View File

@ -1503,6 +1503,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
*out = new(Runc)
(*in).DeepCopyInto(*out)
}
if in.NRI != nil {
in, out := &in.NRI, &out.NRI
*out = new(NRIConfig)
(*in).DeepCopyInto(*out)
}
return
}
@ -4573,6 +4578,37 @@ func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NRIConfig) DeepCopyInto(out *NRIConfig) {
*out = *in
if in.Enabled != nil {
in, out := &in.Enabled, &out.Enabled
*out = new(bool)
**out = **in
}
if in.PluginRegistrationTimeout != nil {
in, out := &in.PluginRegistrationTimeout, &out.PluginRegistrationTimeout
*out = new(v1.Duration)
**out = **in
}
if in.PluginRequestTimeout != nil {
in, out := &in.PluginRequestTimeout, &out.PluginRequestTimeout
*out = new(v1.Duration)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NRIConfig.
func (in *NRIConfig) DeepCopy() *NRIConfig {
if in == nil {
return nil
}
out := new(NRIConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NTPConfig) DeepCopyInto(out *NTPConfig) {
*out = *in