Merge pull request #63409 from mtaufen/kc-validation-feature-gates

Automatic merge from submit-queue (batch tested with PRs 63881, 64046, 63409, 63402, 63221). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Kubelet config: Validate new config against future feature gates

This fixes an issue with KubeletConfiguration validation, where the
feature gates set by the new config were not taken into account.

Also fixes a validation issue with dynamic Kubelet config, where flag
precedence was not enforced prior to dynamic config validation in the
controller; this prevented rejection of dynamic configs that don't merge
well with values set via legacy flags.

Fixes #63305

```release-note
NONE
```

Kubernetes-commit: 6d510f52f266a9a38121435cfd16deb2c45714d7

pkg/util/feature/feature_gate.go

@@ -88,6 +88,10 @@ type FeatureGate interface {
 	Add(features map[Feature]FeatureSpec) error
 	// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
 	KnownFeatures() []string
+	// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
+	// set on the copy without mutating the original. This is useful for validating
+	// config against potential feature gate changes before committing those changes.
+	DeepCopy() FeatureGate
 }
 
 // featureGate implements FeatureGate as well as pflag.Value for flag parsing.
@@ -284,6 +288,10 @@ func (f *featureGate) Enabled(key Feature) bool {
 // AddFlag adds a flag for setting global feature gates to the specified FlagSet.
 func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
 	f.lock.Lock()
+	// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
+	// Not all components expose a feature gates flag using this AddFlag method, and
+	// in the future, all components will completely stop exposing a feature gates flag,
+	// in favor of componentconfig.
 	f.closed = true
 	f.lock.Unlock()
 
@@ -306,3 +314,34 @@ func (f *featureGate) KnownFeatures() []string {
 	sort.Strings(known)
 	return known
 }
+
+// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
+// set on the copy without mutating the original. This is useful for validating
+// config against potential feature gate changes before committing those changes.
+func (f *featureGate) DeepCopy() FeatureGate {
+	// Copy existing state.
+	known := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[k] = v
+	}
+	enabled := map[Feature]bool{}
+	for k, v := range f.enabled.Load().(map[Feature]bool) {
+		enabled[k] = v
+	}
+
+	// Store copied state in new atomics.
+	knownValue := &atomic.Value{}
+	knownValue.Store(known)
+	enabledValue := &atomic.Value{}
+	enabledValue.Store(enabled)
+
+	// Construct a new featureGate around the copied state.
+	// Note that specialFeatures is treated as immutable by convention,
+	// and we maintain the value of f.closed across the copy.
+	return &featureGate{
+		special: specialFeatures,
+		known:   knownValue,
+		enabled: enabledValue,
+		closed:  f.closed,
+	}
+}