From 22612a3528a6ac083dc02bc27aa143c248e0d233 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 19 Jan 2024 16:07:00 -0800 Subject: [PATCH 1/4] apiserver: Add API emulation versioning. Co-authored-by: Siyuan Zhang Co-authored-by: Joe Betz Co-authored-by: Alex Zielenski Signed-off-by: Siyuan Zhang Kubernetes-commit: 403301bfdf2c7312591077827abd2e72f445a53a --- pkg/cel/environment/base.go | 21 +- pkg/features/kube_features.go | 11 + pkg/registry/generic/options.go | 8 +- pkg/registry/generic/registry/store.go | 2 +- pkg/server/config.go | 34 ++- pkg/server/deleted_kinds.go | 65 +++-- pkg/server/deleted_kinds_test.go | 173 +++++++----- pkg/server/genericapiserver.go | 7 + pkg/server/genericapiserver_test.go | 4 +- pkg/server/options/api_enablement_test.go | 42 ++- pkg/server/options/etcd.go | 10 +- pkg/server/options/etcd_test.go | 2 +- pkg/server/options/server_run_options.go | 34 ++- pkg/server/options/server_run_options_test.go | 36 ++- pkg/server/options/serving_test.go | 11 +- .../storage/resource_encoding_config.go | 110 +++++++- pkg/server/storage/storage_factory.go | 19 +- pkg/server/storage/storage_factory_test.go | 246 +++++++++++++++++- pkg/util/feature/feature_gate.go | 2 +- pkg/util/version/registry.go | 142 ++++++++++ pkg/util/version/registry_test.go | 48 ++++ pkg/util/version/version.go | 195 ++++++++++++++ pkg/util/version/version_test.go | 180 +++++++++++++ 23 files changed, 1226 insertions(+), 176 deletions(-) create mode 100644 pkg/util/version/registry.go create mode 100644 pkg/util/version/registry_test.go create mode 100644 pkg/util/version/version.go create mode 100644 pkg/util/version/version_test.go diff --git a/pkg/cel/environment/base.go b/pkg/cel/environment/base.go index 915386999..837cbacdd 100644 --- a/pkg/cel/environment/base.go +++ b/pkg/cel/environment/base.go @@ -30,20 +30,27 @@ import ( "k8s.io/apimachinery/pkg/util/version" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/library" + utilversion "k8s.io/apiserver/pkg/util/version" ) // DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet // that guarantees compatibility with CEL features/libraries/parameters understood by -// an n-1 version +// the api server min compatibility version // -// This default will be set to no more than n-1 the current Kubernetes major.minor version. +// This default will be set to no more than the current Kubernetes major.minor version. // -// Note that a default version number less than n-1 indicates a wider range of version -// compatibility than strictly required for rollback. A wide range of compatibility is -// desirable because it means that CEL expressions are portable across a wider range -// of Kubernetes versions. +// Note that a default version number less than n-1 the current Kubernetes major.minor version +// indicates a wider range of version compatibility than strictly required for rollback. +// A wide range of compatibility is desirable because it means that CEL expressions are portable +// across a wider range of Kubernetes versions. +// A default version number equal to the current Kubernetes major.minor version +// indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - return version.MajorMinor(1, 30) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + if effectiveVer == nil { + effectiveVer = utilversion.DefaultKubeEffectiveVersion() + } + return effectiveVer.MinCompatibilityVersion() } var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 80dc25cc6..8448e36ec 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -322,6 +322,17 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) +} + +// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + // Example: + // EmulationVersion: { + // {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + // }, } // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. diff --git a/pkg/registry/generic/options.go b/pkg/registry/generic/options.go index d675a258f..44d07c0e2 100644 --- a/pkg/registry/generic/options.go +++ b/pkg/registry/generic/options.go @@ -19,6 +19,7 @@ package generic import ( "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -39,12 +40,15 @@ type RESTOptions struct { } // Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests) -func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) { +func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) { return opts, nil } type RESTOptionsGetter interface { - GetRESTOptions(resource schema.GroupResource) (RESTOptions, error) + // GetRESTOptions returns the RESTOptions for the given resource and example object. + // The example object is used to determine the storage version for the resource. + // If the example object is nil, the storage version will be determined by the resource's default storage version. + GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error) } // StoreOptions is set of configuration options used to complete generic registries. diff --git a/pkg/registry/generic/registry/store.go b/pkg/registry/generic/registry/store.go index a8e01708a..98efac933 100644 --- a/pkg/registry/generic/registry/store.go +++ b/pkg/registry/generic/registry/store.go @@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { return err } - opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource) + opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc()) if err != nil { return err } diff --git a/pkg/server/config.go b/pkg/server/config.go index 0502448d6..b266cb2e4 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -42,8 +42,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -70,8 +71,10 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics/features" "k8s.io/component-base/metrics/prometheus/slis" @@ -148,7 +151,12 @@ type Config struct { PostStartHooks map[string]PostStartHookConfigEntry // Version will enable the /version endpoint if non-nil - Version *version.Info + Version *apimachineryversion.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // AuditBackend is where audit events are sent to. AuditBackend audit.Backend // AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request. @@ -585,7 +593,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) { } } -func completeOpenAPI(config *openapicommon.Config, version *version.Info) { +func completeOpenAPI(config *openapicommon.Config, version *version.Version) { if config == nil { return } @@ -624,7 +632,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) { } } -func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) { +func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) { if config == nil { return } @@ -676,6 +684,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { + if c.FeatureGate == nil { + c.FeatureGate = utilfeature.DefaultFeatureGate + } if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { c.ExternalAddress = c.PublicAddress.String() } @@ -691,9 +702,12 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - - completeOpenAPI(c.OpenAPIConfig, c.Version) - completeOpenAPIV3(c.OpenAPIV3Config, c.Version) + var ver *version.Version + if c.EffectiveVersion != nil { + ver = c.EffectiveVersion.EmulationVersion() + } + completeOpenAPI(c.OpenAPIConfig, ver) + completeOpenAPIV3(c.OpenAPIV3Config, ver) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -711,7 +725,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } else { c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string { // use the storage prefix as the key if possible - if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil { + if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil { return opts.ResourcePrefix } // otherwise return "" to use the default key (parent GV name) @@ -819,7 +833,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G APIServerID: c.APIServerID, StorageVersionManager: c.StorageVersionManager, - Version: c.Version, + EffectiveVersion: c.EffectiveVersion, + Version: c.Version, + FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } diff --git a/pkg/server/deleted_kinds.go b/pkg/server/deleted_kinds.go index f14ed9213..841c1f572 100644 --- a/pkg/server/deleted_kinds.go +++ b/pkg/server/deleted_kinds.go @@ -17,6 +17,7 @@ limitations under the License. package server import ( + "fmt" "os" "strconv" "strings" @@ -25,16 +26,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" - apimachineryversion "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/klog/v2" ) // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API type resourceExpirationEvaluator struct { - currentMajor int - currentMinor int - isAlpha bool + currentVersion *apimachineryversion.Version + isAlpha bool // This is usually set for testing for which tests need to be removed. This prevent insta-failing CI. // Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta strictRemovedHandlingInAlpha bool @@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface { ShouldServeForVersion(majorRemoved, minorRemoved int) bool } -func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) { +func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) { + if currentVersion == nil { + return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion") + } + klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion) ret := &resourceExpirationEvaluator{ strictRemovedHandlingInAlpha: false, } - if len(currentVersion.Major) > 0 { - currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32) - if err != nil { - return nil, err - } - ret.currentMajor = int(currentMajor64) - } - if len(currentVersion.Minor) > 0 { - // split the "normal" + and - for semver stuff - minorString := strings.Split(currentVersion.Minor, "+")[0] - minorString = strings.Split(minorString, "-")[0] - minorString = strings.Split(minorString, ".")[0] - currentMinor64, err := strconv.ParseInt(minorString, 10, 32) - if err != nil { - return nil, err - } - ret.currentMinor = int(currentMinor64) - } - - ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha") + // Only keeps the major and minor versions from input version. + ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor()) + ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha") if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok { // do nothing @@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio return false } + introduced, ok := versionedPtr.(introducedInterface) + // skip the introduced check for test where currentVersion is 0.0 + if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if e.currentVersion.LessThan(verIntroduced) { + return false + } + } + removed, ok := versionedPtr.(removedInterface) if !ok { return true @@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool { - if e.currentMajor < majorRemoved { + removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved)) + if removedVer.GreaterThan(e.currentVersion) { return true } - if e.currentMajor > majorRemoved { - return false - } - if e.currentMinor < minorRemoved { - return true - } - if e.currentMinor > minorRemoved { + if removedVer.LessThan(e.currentVersion) { return false } // at this point major and minor are equal, so this API should be removed when the current release GAs. @@ -152,6 +144,11 @@ type removedInterface interface { APILifecycleRemoved() (major, minor int) } +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted. // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage. func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) { @@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi } klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName) + storage := versionToResource[resourceName] + storage.Destroy() delete(versionToResource, resourceName) } versionedResourcesStorageMap[apiVersion] = versionToResource diff --git a/pkg/server/deleted_kinds_test.go b/pkg/server/deleted_kinds_test.go index c033990b4..84403fd15 100644 --- a/pkg/server/deleted_kinds_test.go +++ b/pkg/server/deleted_kinds_test.go @@ -25,57 +25,41 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/dump" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" ) func Test_newResourceExpirationEvaluator(t *testing.T) { tests := []struct { name string - currentVersion version.Info + currentVersion string expected resourceExpirationEvaluator expectedErr string }{ { - name: "beta", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "beta", + currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "alpha", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true}, + name: "alpha", + currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true}, }, { - name: "maintenance", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.1", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "maintenance", + currentVersion: "v1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "bad", - currentVersion: version.Info{ - Major: "1", - Minor: "20something+", - GitVersion: "v1.20.1", - }, - expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`, + name: "no v prefix", + currentVersion: "1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion) + actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion)) checkErr(t, actualErr, tt.expectedErr) if actualErr != nil { @@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) { } } -func storageRemovedIn(major, minor int) removedInStorage { - return removedInStorage{major: major, minor: minor} +func storageRemovedIn(major, minor int) *removedInStorage { + return &removedInStorage{major: major, minor: minor} } -func storageNeverRemoved() removedInStorage { - return removedInStorage{neverRemoved: true} +func storageNeverRemoved() *removedInStorage { + return &removedInStorage{neverRemoved: true} } type removedInStorage struct { @@ -103,23 +87,23 @@ type removedInStorage struct { neverRemoved bool } -func (r removedInStorage) New() runtime.Object { +func (r *removedInStorage) New() runtime.Object { if r.neverRemoved { - return neverRemovedObj{} + return &defaultObj{} } - return removedInObj{major: r.major, minor: r.minor} + return &removedInObj{major: r.major, minor: r.minor} } -func (r removedInStorage) Destroy() { +func (r *removedInStorage) Destroy() { } -type neverRemovedObj struct { +type defaultObj struct { } -func (r neverRemovedObj) GetObjectKind() schema.ObjectKind { +func (r *defaultObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r neverRemovedObj) DeepCopyObject() runtime.Object { +func (r *defaultObj) DeepCopyObject() runtime.Object { panic("don't do this either") } @@ -127,13 +111,45 @@ type removedInObj struct { major, minor int } -func (r removedInObj) GetObjectKind() schema.ObjectKind { +func (r *removedInObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r removedInObj) DeepCopyObject() runtime.Object { +func (r *removedInObj) DeepCopyObject() runtime.Object { panic("don't do this either") } -func (r removedInObj) APILifecycleRemoved() (major, minor int) { +func (r *removedInObj) APILifecycleRemoved() (major, minor int) { + return r.major, r.minor +} + +func storageIntroducedIn(major, minor int) *introducedInStorage { + return &introducedInStorage{major: major, minor: minor} +} + +type introducedInStorage struct { + major, minor int +} + +func (r *introducedInStorage) New() runtime.Object { + if r.major == 0 && r.minor == 0 { + return &defaultObj{} + } + return &IntroducedInObj{major: r.major, minor: r.minor} +} + +func (r *introducedInStorage) Destroy() { +} + +type IntroducedInObj struct { + major, minor int +} + +func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind { + panic("don't do this") +} +func (r *IntroducedInObj) DeepCopyObject() runtime.Object { + panic("don't do this either") +} +func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) { return r.major, r.minor } @@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 20), expected: false, @@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-deferred", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, - isAlpha: true, + currentVersion: apimachineryversion.MajorMinor(1, 20), + isAlpha: true, }, restStorage: storageRemovedIn(1, 20), expected: true, @@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha-but-strict", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true, strictRemovedHandlingInAlpha: true, }, @@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-deferral-does-not-help", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 21, + currentVersion: apimachineryversion.MajorMinor(1, 21), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-major", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 2, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(2, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-future", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 21), expected: true, @@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "never-removed", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageNeverRemoved(), expected: true, }, + { + name: "introduced-in-curr", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 20), + expected: true, + }, + { + name: "introduced-in-prev-major", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 19), + expected: true, + }, + { + name: "introduced-in-future", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 21), + expected: false, + }, + { + name: "missing-introduced", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(0, 0), + expected: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-one-of-two", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-nested-not-expired", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-all-of-version", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { diff --git a/pkg/server/genericapiserver.go b/pkg/server/genericapiserver.go index e0dcbf758..c47fc5b23 100644 --- a/pkg/server/genericapiserver.go +++ b/pkg/server/genericapiserver.go @@ -53,7 +53,9 @@ import ( "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" openapibuilder3 "k8s.io/kube-openapi/pkg/builder3" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -238,6 +240,11 @@ type GenericAPIServer struct { // Version will enable the /version endpoint if non-nil Version *version.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver. lifecycleSignals lifecycleSignals diff --git a/pkg/server/genericapiserver_test.go b/pkg/server/genericapiserver_test.go index 3549b238c..c2bfef4dd 100644 --- a/pkg/server/genericapiserver_test.go +++ b/pkg/server/genericapiserver_test.go @@ -48,6 +48,7 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/registry/rest" genericfilters "k8s.io/apiserver/pkg/server/filters" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/warning" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -460,6 +461,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { kubeVersion := fakeVersion() config.Version = &kubeVersion + config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { @@ -586,7 +588,7 @@ func fakeVersion() version.Info { return version.Info{ Major: "42", Minor: "42", - GitVersion: "42", + GitVersion: "42.42", GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", GitTreeState: "Dirty", BuildDate: time.Now().String(), diff --git a/pkg/server/options/api_enablement_test.go b/pkg/server/options/api_enablement_test.go index b37931eb2..7d6024400 100644 --- a/pkg/server/options/api_enablement_test.go +++ b/pkg/server/options/api_enablement_test.go @@ -32,46 +32,42 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { - name string - testOptions *APIEnablementOptions - expectErr string + name string + testOptions *APIEnablementOptions + runtimeConfig cliflag.ConfigurationMap + expectErr string }{ { name: "test when options is nil", }, { - name: "test when invalid key with only api/all=false", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, - }, - expectErr: "invalid key with only api/all=false", + name: "test when invalid key with only api/all=false", + runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, + expectErr: "invalid key with only api/all=false", }, { - name: "test when ConfigurationMap key is invalid", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, - }, - expectErr: "runtime-config invalid key", + name: "test when ConfigurationMap key is invalid", + runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, + expectErr: "runtime-config invalid key", }, { - name: "test when unknown api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, - }, - expectErr: "unknown api groups", + name: "test when unknown api groups", + runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, + expectErr: "unknown api groups", }, { - name: "test when valid api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, - }, + name: "test when valid api groups", + runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, }, } testGroupRegistry := fakeGroupRegistry{} for _, testcase := range testCases { t.Run(testcase.name, func(t *testing.T) { - errs := testcase.testOptions.Validate(testGroupRegistry) + testOptions := &APIEnablementOptions{ + RuntimeConfig: testcase.runtimeConfig, + } + errs := testOptions.Validate(testGroupRegistry) if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) { t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr) } diff --git a/pkg/server/options/etcd.go b/pkg/server/options/etcd.go index 10f9775ef..af7696c40 100644 --- a/pkg/server/options/etcd.go +++ b/pkg/server/options/etcd.go @@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct { StorageFactory serverstorage.StorageFactory } -func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := f.StorageFactory.NewConfig(resource) +func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + storageConfig, err := f.StorageFactory.NewConfig(resource, example) if err != nil { return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error()) } @@ -469,7 +469,7 @@ type SimpleStorageFactory struct { StorageConfig storagebackend.Config } -func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { return s.StorageConfig.ForResource(resource), nil } @@ -493,8 +493,8 @@ type transformerStorageFactory struct { resourceTransformers storagevalue.ResourceTransformers } -func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { - config, err := t.delegate.NewConfig(resource) +func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { + config, err := t.delegate.NewConfig(resource, example) if err != nil { return nil, err } diff --git a/pkg/server/options/etcd_test.go b/pkg/server/options/etcd_test.go index 06d0cae0a..ea4464b24 100644 --- a/pkg/server/options/etcd_test.go +++ b/pkg/server/options/etcd_test.go @@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) { if err := etcdOptions.ApplyTo(serverConfig); err != nil { t.Fatalf("Failed to apply etcd options error: %v", err) } - restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}) + restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/server/options/server_run_options.go b/pkg/server/options/server_run_options.go index 1373d8a4d..88dc3802f 100644 --- a/pkg/server/options/server_run_options.go +++ b/pkg/server/options/server_run_options.go @@ -26,7 +26,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/server" - utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -89,9 +90,13 @@ type ServerRunOptions struct { // This grace period is orthogonal to other grace periods, and // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration + + // FeatureGate are the featuregate to install on the CLI + FeatureGate featuregate.FeatureGate + EffectiveVersion utilversion.EffectiveVersion } -func NewServerRunOptions() *ServerRunOptions { +func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -104,6 +109,8 @@ func NewServerRunOptions() *ServerRunOptions { JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, } } @@ -124,6 +131,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod + c.EffectiveVersion = s.EffectiveVersion + c.FeatureGate = s.FeatureGate return nil } @@ -196,6 +205,14 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } + if s.FeatureGate != nil { + if errs := s.FeatureGate.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } + } + if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } return errors } @@ -336,6 +353,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") - - utilfeature.DefaultMutableFeatureGate.AddFlag(fs) +} + +// Complete fills missing fields with defaults. +func (s *ServerRunOptions) Complete() error { + if s.FeatureGate == nil { + return fmt.Errorf("nil FeatureGate in ServerRunOptions") + } + if s.EffectiveVersion == nil { + return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") + } + return nil } diff --git a/pkg/server/options/server_run_options_test.go b/pkg/server/options/server_run_options_test.go index d1f67fe07..0c62a4d68 100644 --- a/pkg/server/options/server_run_options_test.go +++ b/pkg/server/options/server_run_options_test.go @@ -23,10 +23,14 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") testCases := []struct { name string testOptions *ServerRunOptions @@ -43,6 +47,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -57,6 +63,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -71,6 +79,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--request-timeout can not be negative value", }, @@ -85,6 +95,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -99,6 +111,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -113,6 +127,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -128,6 +144,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -143,6 +161,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -158,6 +178,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, @@ -173,6 +195,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, }, } @@ -192,6 +216,8 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -239,7 +265,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -263,6 +289,8 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -271,13 +299,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions() + return NewServerRunOptions(featureGate, effectiveVersion) }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions() + o := NewServerRunOptions(featureGate, effectiveVersion) o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -304,7 +332,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/pkg/server/options/serving_test.go b/pkg/server/options/serving_test.go index a08ce2b3b..40feeaf62 100644 --- a/pkg/server/options/serving_test.go +++ b/pkg/server/options/serving_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/server" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery" restclient "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" @@ -276,9 +277,9 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) - v := fakeVersion() config.Version = &v + config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true secureOptions := (&SecureServingOptions{ @@ -463,11 +464,9 @@ func certSignature(cert tls.Certificate) (string, error) { func fakeVersion() version.Info { return version.Info{ - Major: "42", - Minor: "42", - GitVersion: "42", - GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", - GitTreeState: "Dirty", + Major: "42", + Minor: "42", + GitVersion: "42.42", } } diff --git a/pkg/server/storage/resource_encoding_config.go b/pkg/server/storage/resource_encoding_config.go index efb22fbc8..5d5cfccc7 100644 --- a/pkg/server/storage/resource_encoding_config.go +++ b/pkg/server/storage/resource_encoding_config.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/util/version" ) type ResourceEncodingConfig interface { @@ -33,10 +35,15 @@ type ResourceEncodingConfig interface { InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error) } +type CompatibilityResourceEncodingConfig interface { + BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error) +} + type DefaultResourceEncodingConfig struct { // resources records the overriding encoding configs for individual resources. - resources map[schema.GroupResource]*OverridingResourceEncoding - scheme *runtime.Scheme + resources map[schema.GroupResource]*OverridingResourceEncoding + scheme *runtime.Scheme + effectiveVersion version.EffectiveVersion } type OverridingResourceEncoding struct { @@ -47,7 +54,7 @@ type OverridingResourceEncoding struct { var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{} func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig { - return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme} + return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()} } func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) { @@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored } } +func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) { + o.effectiveVersion = effectiveVersion +} + func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil } +func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) { + if !o.scheme.IsGroupRegistered(resource.Group) { + return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) + } + + // Always respect overrides + resourceOverride, resourceExists := o.resources[resource] + if resourceExists { + return resourceOverride.ExternalResourceEncoding, nil + } + + return emulatedStorageVersion( + o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], + example, + o.effectiveVersion, + o.scheme) +} + func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -82,3 +111,78 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou } return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil } + +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + +func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) { + if example == nil || effectiveVersion == nil { + return binaryVersionOfResource, nil + } + + // Look up example in scheme to find all objects of the same Group-Kind + // Use the highest priority version for that group-kind whose lifecycle window + // includes the current emulation version. + // If no version is found, use the binary version + // (in this case the API should be disabled anyway) + gvks, _, err := scheme.ObjectKinds(example) + if err != nil { + return schema.GroupVersion{}, err + } else if len(gvks) == 0 { + // Probably shouldn't happen if err is non-nil + return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example) + } + + // VersionsForGroupKind returns versions in priority order + versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind}) + + compatibilityVersion := effectiveVersion.MinCompatibilityVersion() + + for _, gv := range versions { + if gv.Version == runtime.APIVersionInternal { + continue + } + + gvk := schema.GroupVersionKind{ + Group: gv.Group, + Version: gv.Version, + Kind: gvks[0].Kind, + } + + exampleOfGVK, err := scheme.New(gvk) + if err != nil { + return schema.GroupVersion{}, err + } + + // If it was introduced after current compatibility version, don't use it + if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { + // API resource lifecycles should be relative to k8s api version + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if introducedVer.GreaterThan(compatibilityVersion) { + continue + } + } + + // versions is returned in priority order, so just use first result + return gvk.GroupVersion(), nil + } + + // Getting here means we're serving a version that is unknown to the + // min-compatibility-version server. + // + // This is only expected to happen when serving an alpha API type due + // to missing pre-release lifecycle information + // (which doesn't happen by default), or when emulation-version and + // min-compatibility-version are several versions apart so a beta or GA API + // was being served which didn't exist at all in min-compatibility-version. + // + // In the alpha case - we do not support compatibility versioning of + // alpha types and recommend users do not mix the two. + // In the skip-level case - The version of apiserver we are retaining + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. + return binaryVersionOfResource, nil +} diff --git a/pkg/server/storage/storage_factory.go b/pkg/server/storage/storage_factory.go index 0dc50cea6..ad01c5a5d 100644 --- a/pkg/server/storage/storage_factory.go +++ b/pkg/server/storage/storage_factory.go @@ -42,7 +42,7 @@ type Backend struct { type StorageFactory interface { // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. - NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) + NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) // ResourcePrefix returns the overridden resource prefix for the GroupResource // This allows for cohabitation of resources with different native types and provides @@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. -func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { chosenStorageResource := s.getStorageGroupResource(groupResource) // operate on copy @@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* } var err error - codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) - if err != nil { - return nil, err + if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok { + codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example) + if err != nil { + return nil, err + } + } else { + codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) + if err != nil { + return nil, err + } } + codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) if err != nil { return nil, err } + codecConfig.Config = storageConfig storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) diff --git a/pkg/server/storage/storage_factory_test.go b/pkg/server/storage/storage_factory_test.go index c52049ca0..a12e865f0 100644 --- a/pkg/server/storage/storage_factory_test.go +++ b/pkg/server/storage/storage_factory_test.go @@ -26,10 +26,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/apis/example" exampleinstall "k8s.io/apiserver/pkg/apis/example/install" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/version" ) var ( @@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) { f.SetEtcdLocation(example.Resource("*"), []string{"/server2"}) f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test") - config, err := f.NewConfig(example.Resource("test")) + config, err := f.NewConfig(example.Resource("test"), nil) if err != nil { t.Fatal(err) } @@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { storageFactory.SetEtcdLocation(test.resource, test.servers) var err error - config, err := storageFactory.NewConfig(test.resource) + config, err := storageFactory.NewConfig(test.resource, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { continue } - config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}) + config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) { } } } + +var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{} +var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{} + +type fakeLifecycler[T, V any] struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +type removedLifecycler[T, V any] struct { + fakeLifecycler[T, V] +} + +func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f } +func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f } +func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) { + if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok { + return int(introduced.Major()), int(introduced.Minor()) + } + panic("no lifecycle version set") +} +func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) { + if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok { + return int(removed.Major()), int(removed.Minor()) + } + panic("no lifecycle version set") +} + +func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) { + f := fakeLifecycler[T, V]{} + + introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced) + + var res runtime.Object + if removed != "" { + removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed) + res = &removedLifecycler[T, V]{fakeLifecycler: f} + } else { + res = &f + } + + var v V + var t T + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: strings.ToLower(reflect.TypeOf(v).Name()), + Kind: reflect.TypeOf(t).Name(), + }, + res, + ) + + // Also ensure internal version is registered + // If it is registertd multiple times, it will ignore subsequent registrations + internalInstance := &fakeLifecycler[T, struct{}]{} + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: runtime.APIVersionInternal, + Kind: reflect.TypeOf(t).Name(), + }, + internalInstance, + ) +} + +func TestStorageFactoryCompatibilityVersion(t *testing.T) { + // Creates a scheme with stub types for unit test + sch := runtime.NewScheme() + codecs := serializer.NewCodecFactory(sch) + + type Internal = struct{} + type V1beta1 struct{} + type V1beta2 struct{} + type V1beta3 struct{} + type V1 struct{} + + type Pod struct{} + type FlowSchema struct{} + type ValidatingAdmisisonPolicy struct{} + type CronJob struct{} + + // Order dictates priority order + registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "") + registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0") + registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0") + registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0") + registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "") + registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0") + registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "") + + // FlowSchema + // - v1beta1: 1.20.0 - 1.23.0 + // - v1beta2: 1.23.0 - 1.26.0 + // - v1beta3: 1.26.0 - 1.30.0 + // - v1: 1.29.0+ + // CronJob + // - v1beta1: 1.8.0 - 1.21.0 + // - v1: 1.21.0+ + // ValidatingAdmissionPolicy + // - v1beta1: 1.28.0 - 1.31.0 + // - v1: 1.30.0+ + + testcases := []struct { + effectiveVersion string + example runtime.Object + expectedVersion schema.GroupVersion + }{ + { + // Basic case. Beta version for long time + effectiveVersion: "1.14.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. Beta version for long time + effectiveVersion: "1.20.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. GA version for long time + effectiveVersion: "1.28.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"}, + }, + { + // Basic core/v1 + effectiveVersion: "1.31.0", + example: &fakeLifecycler[Pod, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "", Version: "v1"}, + }, + { + // Corner case: 1.1.0 has no flowcontrol. Options are to error + // out or to use the latest version. This test assumes the latter. + effectiveVersion: "1.1.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.21.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + // v2Beta1 introduced this version, but minCompatibility should + // force v1beta1 + effectiveVersion: "1.23.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.24.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.26.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.27.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // GA API introduced 1.29 but must keep storing in v1beta3 for downgrades + effectiveVersion: "1.29.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // Version after GA api is introduced + effectiveVersion: "1.30.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.30.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.31.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.29.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + } + + for _, tc := range testcases { + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvk := gvks[0] + t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) { + config := NewDefaultResourceEncodingConfig(sch) + config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion)) + f := NewDefaultStorageFactory( + storagebackend.Config{}, + "", + codecs, + config, + NewResourceConfig(), + nil) + + cfg, err := f.NewConfig(schema.GroupResource{ + Group: gvk.Group, + Resource: gvk.Kind, // doesnt really matter here + }, tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion, + schema.GroupKind{ + Group: gvks[0].Group, + }, schema.GroupKind{ + Group: gvks[0].Group, + }) + if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() { + t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner) + } + }) + } +} diff --git a/pkg/util/feature/feature_gate.go b/pkg/util/feature/feature_gate.go index 7dd3df589..00a9e099b 100644 --- a/pkg/util/feature/feature_gate.go +++ b/pkg/util/feature/feature_gate.go @@ -25,7 +25,7 @@ var ( // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. // Tests that need to modify feature gates for the duration of their test should use: // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., ) - DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() + DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate() // DefaultFeatureGate is a shared global FeatureGate. // Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate. diff --git a/pkg/util/version/registry.go b/pkg/util/version/registry.go new file mode 100644 index 000000000..f8b4ef5cd --- /dev/null +++ b/pkg/util/version/registry.go @@ -0,0 +1,142 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "sync" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/featuregate" +) + +var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() + +const ( + ComponentGenericAPIServer = "k8s.io/apiserver" +) + +// ComponentGlobals stores the global variables for a component for easy access. +type ComponentGlobals struct { + effectiveVersion MutableEffectiveVersion + featureGate featuregate.MutableVersionedFeatureGate +} + +type ComponentGlobalsRegistry interface { + // EffectiveVersionFor returns the EffectiveVersion registered under the component. + // Returns nil if the component is not registered. + EffectiveVersionFor(component string) EffectiveVersion + // FeatureGateFor returns the FeatureGate registered under the component. + // Returns nil if the component is not registered. + FeatureGateFor(component string) featuregate.FeatureGate + // Register registers the EffectiveVersion and FeatureGate for a component. + // Overrides existing ComponentGlobals if it is already in the registry if override is true, + // otherwise returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. + // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. + ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) + // SetAllComponents sets the emulation version for other global variables for all components registered. + SetAllComponents() error + // SetAllComponents calls the Validate() function for all the global variables for all components registered. + ValidateAllComponents() []error +} + +type componentGlobalsRegistry struct { + componentGlobals map[string]ComponentGlobals + mutex sync.RWMutex +} + +func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { + return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} +} + +func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.effectiveVersion +} + +func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.featureGate +} + +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if _, ok := r.componentGlobals[component]; ok && !override { + return fmt.Errorf("component globals of %s already registered", component) + } + if featureGate != nil { + featureGate.DeferErrorsToValidation(true) + } + c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} + r.componentGlobals[component] = c + return nil +} + +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.unsafeRegister(component, effectiveVersion, featureGate, override) +} + +func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { + r.mutex.Lock() + defer r.mutex.Unlock() + globals, ok := r.componentGlobals[component] + if ok { + return globals.effectiveVersion, globals.featureGate + } + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + return effectiveVersion, featureGate +} + +func (r *componentGlobalsRegistry) SetAllComponents() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { + return err + } + } + return nil +} + +func (r *componentGlobalsRegistry) ValidateAllComponents() []error { + var errs []error + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + errs = append(errs, globals.effectiveVersion.Validate()...) + if globals.featureGate != nil { + errs = append(errs, globals.featureGate.Validate()...) + } + } + return errs +} diff --git a/pkg/util/version/registry_test.go b/pkg/util/version/registry_test.go new file mode 100644 index 000000000..2e1955270 --- /dev/null +++ b/pkg/util/version/registry_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "testing" +) + +func TestEffectiveVersionRegistry(t *testing.T) { + r := NewComponentGlobalsRegistry() + testComponent := "test" + ver1 := NewEffectiveVersion("1.31") + ver2 := NewEffectiveVersion("1.28") + + if r.EffectiveVersionFor(testComponent) != nil { + t.Fatalf("expected nil EffectiveVersion initially") + } + if err := r.Register(testComponent, ver1, nil, false); err != nil { + t.Fatalf("expected no error to register new component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { + t.Fatalf("expected EffectiveVersionFor to return the version registered") + } + // overwrite + if err := r.Register(testComponent, ver2, nil, false); err == nil { + t.Fatalf("expected error to register existing component when override is false") + } + if err := r.Register(testComponent, ver2, nil, true); err != nil { + t.Fatalf("expected no error to overriding existing component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + t.Fatalf("expected EffectiveVersionFor to return the version overridden") + } +} diff --git a/pkg/util/version/version.go b/pkg/util/version/version.go new file mode 100644 index 000000000..fce927c6c --- /dev/null +++ b/pkg/util/version/version.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + baseversion "k8s.io/component-base/version" +) + +type EffectiveVersion interface { + BinaryVersion() *version.Version + EmulationVersion() *version.Version + MinCompatibilityVersion() *version.Version + EqualTo(other EffectiveVersion) bool + String() string + Validate() []error +} + +type MutableEffectiveVersion interface { + EffectiveVersion + Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) + SetEmulationVersion(emulationVersion *version.Version) + SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) + // AddFlags adds the "{prefix}-emulated-version" to the flagset. + AddFlags(fs *pflag.FlagSet, prefix string) +} + +type VersionVar struct { + val atomic.Pointer[version.Version] +} + +// Set sets the flag value +func (v *VersionVar) Set(s string) error { + components := strings.Split(s, ".") + if len(components) != 2 { + return fmt.Errorf("version %s is not in the format of major.minor", s) + } + ver, err := version.ParseGeneric(s) + if err != nil { + return err + } + v.val.Store(ver) + return nil +} + +// String returns the flag value +func (v *VersionVar) String() string { + ver := v.val.Load() + return ver.String() +} + +// Type gets the flag type +func (v *VersionVar) Type() string { + return "version" +} + +type effectiveVersion struct { + binaryVersion atomic.Pointer[version.Version] + // If the emulationVersion is set by the users, it could only contain major and minor versions. + // In tests, emulationVersion could be the same as the binary version, or set directly, + // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + emulationVersion VersionVar + // minCompatibilityVersion could only contain major and minor versions. + minCompatibilityVersion VersionVar +} + +func (m *effectiveVersion) BinaryVersion() *version.Version { + return m.binaryVersion.Load() +} + +func (m *effectiveVersion) EmulationVersion() *version.Version { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) +} + +func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { + return m.minCompatibilityVersion.val.Load() +} + +func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { + return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion()) +} + +func (m *effectiveVersion) String() string { + if m == nil { + return "" + } + return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}", + m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) +} + +func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { + m.binaryVersion.Store(binaryVersion) + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) +} + +func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) Validate() []error { + var errs []error + // Validate only checks the major and minor versions. + binaryVersion := m.binaryVersion.Load().WithPatch(0) + emulationVersion := m.emulationVersion.val.Load() + minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + + // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. + maxEmuVer := binaryVersion + minEmuVer := binaryVersion.SubtractMinor(1) + // TODO: remove in 1.32 + // emulationVersion is introduced in 1.31, so it cannot be lower than that. + // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { + minEmuVer = version.MajorMinor(1, 31) + } + if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { + errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) + } + // minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha. + maxCompVer := binaryVersion.SubtractMinor(1) + minCompVer := binaryVersion.SubtractMinor(1) + if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) { + errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String())) + } + return errs +} + +// AddFlags adds the "{prefix}-emulated-version" to the flagset. +func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { + if m == nil { + return + } + if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ + "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ + "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ + "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ + "Format could only be major.minor") +} + +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + effective := &effectiveVersion{} + binaryVersion := version.MustParse(binaryVer) + compatVersion := binaryVersion.SubtractMinor(1) + effective.Set(binaryVersion, binaryVersion, compatVersion) + return effective +} + +// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the +// current build information. +func DefaultBuildEffectiveVersion() MutableEffectiveVersion { + verInfo := baseversion.Get() + ver := NewEffectiveVersion(verInfo.String()) + if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { + ver = DefaultKubeEffectiveVersion() + } + return ver +} + +// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the +// latest K8s release. +// Should update for each minor release! +func DefaultKubeEffectiveVersion() MutableEffectiveVersion { + return NewEffectiveVersion("1.31") +} diff --git a/pkg/util/version/version_test.go b/pkg/util/version/version_test.go new file mode 100644 index 000000000..8c61e92c1 --- /dev/null +++ b/pkg/util/version/version_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + minCompatibilityVersion string + expectErrors bool + }{ + { + name: "patch version diff ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.1", + minCompatibilityVersion: "v1.31.5", + }, + { + name: "emulation version one minor lower than binary ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.31.0", + }, + { + name: "binary version 1.31, emulation version lower than 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "binary version 1.31, emulation version 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.30.0", + }, + { + name: "binary version lower than 1.31", + binaryVersion: "v1.30.2", + emulationVersion: "v1.29.0", + minCompatibilityVersion: "v1.29.0", + }, + { + name: "emulation version two minor lower than binary not ok", + binaryVersion: "v1.33.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "emulation version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.33.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "emulation version two minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.34.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "compatibility version same as binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "compatibility version two minor lower than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "compatibility version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.33.0", + expectErrors: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binaryVersion := version.MustParseGeneric(test.binaryVersion) + effective := &effectiveVersion{} + emulationVersion := version.MustParseGeneric(test.emulationVersion) + minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) + effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion) + + errs := effective.Validate() + if len(errs) > 0 && !test.expectErrors { + t.Errorf("expected no errors, errors found %+v", errs) + } + + if len(errs) == 0 && test.expectErrors { + t.Errorf("expected errors, no errors found") + } + }) + } +} + +func TestEffectiveVersionsFlag(t *testing.T) { + tests := []struct { + name string + emulationVerson string + expectedEmulationVersion *version.Version + parseError string + }{ + { + name: "major.minor ok", + emulationVerson: "1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "v prefix ok", + emulationVerson: "v1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "semantic version not ok", + emulationVerson: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", + }, + { + name: "invalid version", + emulationVerson: "1.foo", + parseError: "illegal version string", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + effective := NewEffectiveVersion("1.30") + effective.AddFlags(fs, "test") + + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) + } + if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { + t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) + } + }) + } +} From c80af88d2144ad6dac9229e2b69ccc5334c51f0c Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Thu, 30 May 2024 12:08:52 -0700 Subject: [PATCH 2/4] Add composition flags for emulation version and feature gate. Signed-off-by: Siyuan Zhang Kubernetes-commit: 701e5fc3744c2b21ba5b1ca3399b71c9becf06d0 --- pkg/cel/environment/base.go | 2 +- pkg/util/version/registry.go | 153 ++++++++++++++++++-- pkg/util/version/registry_test.go | 225 +++++++++++++++++++++++++++++- pkg/util/version/version_test.go | 20 +-- 4 files changed, 379 insertions(+), 21 deletions(-) diff --git a/pkg/cel/environment/base.go b/pkg/cel/environment/base.go index 837cbacdd..5e2e995c1 100644 --- a/pkg/cel/environment/base.go +++ b/pkg/cel/environment/base.go @@ -46,7 +46,7 @@ import ( // A default version number equal to the current Kubernetes major.minor version // indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) if effectiveVer == nil { effectiveVer = utilversion.DefaultKubeEffectiveVersion() } diff --git a/pkg/util/version/registry.go b/pkg/util/version/registry.go index f8b4ef5cd..589d42a1a 100644 --- a/pkg/util/version/registry.go +++ b/pkg/util/version/registry.go @@ -18,16 +18,47 @@ package version import ( "fmt" + "sort" + "strings" "sync" + "github.com/spf13/pflag" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" + "k8s.io/klog/v2" ) +// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. +// Example usage: +// // register the component effective version and feature gate first +// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) +// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") +// wardleFeatureGate := featuregate.NewFeatureGate() +// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) +// +// cmd := &cobra.Command{ +// ... +// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE +// PersistentPreRunE: func(*cobra.Command, []string) error { +// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { +// return err +// } +// ... +// }, +// RunE: func(c *cobra.Command, args []string) error { +// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere +// }, +// } +// +// flags := cmd.Flags() +// // add flags +// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() const ( - ComponentGenericAPIServer = "k8s.io/apiserver" + DefaultKubeComponent = "kube" ) // ComponentGlobals stores the global variables for a component for easy access. @@ -50,19 +81,27 @@ type ComponentGlobalsRegistry interface { // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) - // SetAllComponents sets the emulation version for other global variables for all components registered. - SetAllComponents() error + // AddFlags adds flags of "--emulated-version" and "--feature-gates" + AddFlags(fs *pflag.FlagSet) + // Set sets the flags for all global variables for all components registered. + Set() error // SetAllComponents calls the Validate() function for all the global variables for all components registered. - ValidateAllComponents() []error + Validate() []error } type componentGlobalsRegistry struct { componentGlobals map[string]ComponentGlobals mutex sync.RWMutex + // map of component name to emulation version set from the flag. + emulationVersionConfig cliflag.ConfigurationMap + // map of component name to the list of feature gates set from the flag. + featureGatesConfig map[string][]string } func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { - return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} + return &componentGlobalsRegistry{ + componentGlobals: make(map[string]ComponentGlobals), + } } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -90,7 +129,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { - featureGate.DeferErrorsToValidation(true) + if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil { + return err + } } c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} r.componentGlobals[component] = c @@ -98,6 +139,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer } func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if effectiveVersion == nil { + return fmt.Errorf("cannot register nil effectiveVersion") + } r.mutex.Lock() defer r.mutex.Unlock() return r.unsafeRegister(component, effectiveVersion, featureGate, override) @@ -114,21 +158,112 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) SetAllComponents() error { +func (r *componentGlobalsRegistry) knownFeatures() []string { r.mutex.Lock() defer r.mutex.Unlock() - for _, globals := range r.componentGlobals { + var known []string + for component, globals := range r.componentGlobals { if globals.featureGate == nil { continue } + for _, f := range globals.featureGate.KnownFeatures() { + known = append(known, component+":"+f) + } + } + sort.Strings(known) + return known +} + +func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { + r.mutex.Lock() + defer r.mutex.Unlock() + var vs []string + for component, globals := range r.componentGlobals { + binaryVer := globals.effectiveVersion.BinaryVersion() + if isEmulation { + // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} + // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) + } else { + // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) + } + } + sort.Strings(vs) + return vs +} + +func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { + if r == nil { + return + } + r.mutex.Lock() + for _, globals := range r.componentGlobals { + if globals.featureGate != nil { + globals.featureGate.Close() + } + } + r.emulationVersionConfig = make(cliflag.ConfigurationMap) + r.featureGatesConfig = make(map[string][]string) + r.mutex.Unlock() + + fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + + fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ + "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ + "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) +} + +func (r *componentGlobalsRegistry) Set() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for comp, emuVer := range r.emulationVersionConfig { + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) + v, err := version.Parse(emuVer) + if err != nil { + return err + } + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) + } + // Set feature gate emulation version before setting feature gate flag values. + for comp, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } + for comp, fg := range r.featureGatesConfig { + if comp == "" { + comp = DefaultKubeComponent + } + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + featureGate := r.componentGlobals[comp].featureGate + if featureGate == nil { + return fmt.Errorf("component featureGate not registered: %s", comp) + } + flagVal := strings.Join(fg, ",") + klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + if err := featureGate.Set(flagVal); err != nil { + return err + } + } return nil } -func (r *componentGlobalsRegistry) ValidateAllComponents() []error { +func (r *componentGlobalsRegistry) Validate() []error { var errs []error r.mutex.Lock() defer r.mutex.Unlock() diff --git a/pkg/util/version/registry_test.go b/pkg/util/version/registry_test.go index 2e1955270..68625e526 100644 --- a/pkg/util/version/registry_test.go +++ b/pkg/util/version/registry_test.go @@ -17,12 +17,22 @@ limitations under the License. package version import ( + "fmt" + "strings" "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" +) + +const ( + testComponent = "test" ) func TestEffectiveVersionRegistry(t *testing.T) { r := NewComponentGlobalsRegistry() - testComponent := "test" ver1 := NewEffectiveVersion("1.31") ver2 := NewEffectiveVersion("1.28") @@ -46,3 +56,216 @@ func TestEffectiveVersionRegistry(t *testing.T) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } + +func testRegistry(t *testing.T) *componentGlobalsRegistry { + r := componentGlobalsRegistry{ + componentGlobals: map[string]ComponentGlobals{}, + emulationVersionConfig: make(cliflag.ConfigurationMap), + featureGatesConfig: make(map[string][]string), + } + verKube := NewEffectiveVersion("1.31") + fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "kubeA": { + {Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "kubeB": { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + + verTest := NewEffectiveVersion("2.8") + fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "testA": { + {Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + "testB": { + {Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) + _ = r.Register(testComponent, verTest, fgTest, true) + return &r +} + +func TestVersionFlagOptions(t *testing.T) { + r := testRegistry(t) + emuVers := strings.Join(r.versionFlagOptions(true), "\n") + expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionedFeatureGateFlag(t *testing.T) { + r := testRegistry(t) + known := strings.Join(r.knownFeatures(), "\n") + expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + + "kube:AllBeta=true|false (BETA - default=false)\n" + + "kube:commonC=true|false (BETA - default=true)\n" + + "kube:kubeB=true|false (ALPHA - default=false)\n" + + "test:AllAlpha=true|false (ALPHA - default=false)\n" + + "test:AllBeta=true|false (BETA - default=false)\n" + + "test:commonC=true|false (ALPHA - default=false)\n" + + "test:testA=true|false (BETA - default=false)" + if known != expectedKnown { + t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known) + } +} + +func TestFlags(t *testing.T) { + tests := []struct { + name string + emulationVersionFlag string + featureGatesFlag string + parseError string + expectedKubeEmulationVersion *version.Version + expectedTestEmulationVersion *version.Version + expectedKubeFeatureValues map[featuregate.Feature]bool + expectedTestFeatureValues map[featuregate.Feature]bool + }{ + { + name: "setting kube emulation version", + emulationVersionFlag: "kube=1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting kube emulation version, prefix v ok", + emulationVersionFlag: "kube=v1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting test emulation version", + emulationVersionFlag: "test=2.7", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + }, + { + name: "version missing component", + emulationVersionFlag: "1.31", + parseError: "component not registered: 1.31", + }, + { + name: "version unregistered component", + emulationVersionFlag: "test3=1.31", + parseError: "component not registered: test3", + }, + { + name: "invalid version", + emulationVersionFlag: "test=1.foo", + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, + }, + { + name: "setting future test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true,test:testB=true", + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + }, + { + name: "setting kube feature flag", + emulationVersionFlag: "test=2.7,kube=1.30", + featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, + }, + { + name: "setting locked kube feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "kubeA=false", + parseError: "cannot set feature gate kubeA to false, feature is locked to true", + }, + { + name: "setting unknown test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testD=true", + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test3:commonC=true", + parseError: "component not registered: test3", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r := testRegistry(t) + r.AddFlags(fs) + + err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), + fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + if err == nil { + err = r.Set() + } + if test.parseError != "" { + if err == nil || !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) + } + if test.expectedKubeEmulationVersion != nil { + v := r.EffectiveVersionFor("kube").EmulationVersion() + if !v.EqualTo(test.expectedKubeEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) + return + } + } + if test.expectedTestEmulationVersion != nil { + v := r.EffectiveVersionFor("test").EmulationVersion() + if !v.EqualTo(test.expectedTestEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) + return + } + } + for f, v := range test.expectedKubeFeatureValues { + if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { + t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v) + } + } + for f, v := range test.expectedTestFeatureValues { + if r.FeatureGateFor(testComponent).Enabled(f) != v { + t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v) + } + } + }) + } +} diff --git a/pkg/util/version/version_test.go b/pkg/util/version/version_test.go index 8c61e92c1..24db0318f 100644 --- a/pkg/util/version/version_test.go +++ b/pkg/util/version/version_test.go @@ -131,29 +131,29 @@ func TestValidate(t *testing.T) { func TestEffectiveVersionsFlag(t *testing.T) { tests := []struct { name string - emulationVerson string + emulationVersion string expectedEmulationVersion *version.Version parseError string }{ { name: "major.minor ok", - emulationVerson: "1.30", + emulationVersion: "1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { name: "v prefix ok", - emulationVerson: "v1.30", + emulationVersion: "v1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { - name: "semantic version not ok", - emulationVerson: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", + name: "semantic version not ok", + emulationVersion: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", }, { - name: "invalid version", - emulationVerson: "1.foo", - parseError: "illegal version string", + name: "invalid version", + emulationVersion: "1.foo", + parseError: "illegal version string", }, } for i, test := range tests { @@ -162,7 +162,7 @@ func TestEffectiveVersionsFlag(t *testing.T) { effective := NewEffectiveVersion("1.30") effective.AddFlags(fs, "test") - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) From 00857ca9ec84204bf59080cbc04cbf68d3edee7c Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 31 May 2024 20:29:48 -0700 Subject: [PATCH 3/4] Add version mapping in ComponentGlobalsRegistry. Signed-off-by: Siyuan Zhang Kubernetes-commit: 4352c4ad2762ce49ce30e62381f8ceb24723fbcc --- pkg/server/config.go | 14 +- pkg/server/config_test.go | 3 + pkg/server/deleted_kinds.go | 2 +- pkg/server/genericapiserver.go | 3 - pkg/server/genericapiserver_test.go | 7 +- pkg/server/options/serving_test.go | 1 - .../storage/resource_encoding_config.go | 1 + pkg/util/version/registry.go | 229 +++++++++++-- pkg/util/version/registry_test.go | 307 +++++++++++++----- pkg/util/version/version.go | 107 +++--- pkg/util/version/version_test.go | 54 --- 11 files changed, 471 insertions(+), 257 deletions(-) diff --git a/pkg/server/config.go b/pkg/server/config.go index b266cb2e4..931226bb6 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -44,7 +44,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -150,8 +149,6 @@ type Config struct { // done values in this values for this map are ignored. PostStartHooks map[string]PostStartHookConfigEntry - // Version will enable the /version endpoint if non-nil - Version *apimachineryversion.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion @@ -702,12 +699,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - var ver *version.Version - if c.EffectiveVersion != nil { - ver = c.EffectiveVersion.EmulationVersion() - } - completeOpenAPI(c.OpenAPIConfig, ver) - completeOpenAPIV3(c.OpenAPIV3Config, ver) + completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion()) + completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion()) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -834,7 +827,6 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G StorageVersionManager: c.StorageVersionManager, EffectiveVersion: c.EffectiveVersion, - Version: c.Version, FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, @@ -1103,7 +1095,7 @@ func installAPI(s *GenericAPIServer, c *Config) { } } - routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer) + routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { diff --git a/pkg/server/config_test.go b/pkg/server/config_test.go index f58f3bf9c..6d25272b0 100644 --- a/pkg/server/config_test.go +++ b/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" @@ -90,6 +91,7 @@ func TestNewWithDelegate(t *testing.T) { delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") delegateConfig.LoopbackClientConfig = &rest.Config{} + delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") clientset := fake.NewSimpleClientset() if clientset == nil { t.Fatal("unable to create fake client set") @@ -122,6 +124,7 @@ func TestNewWithDelegate(t *testing.T) { wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} + wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error { return fmt.Errorf("wrapping failed healthcheck") diff --git a/pkg/server/deleted_kinds.go b/pkg/server/deleted_kinds.go index 841c1f572..564b5bc43 100644 --- a/pkg/server/deleted_kinds.go +++ b/pkg/server/deleted_kinds.go @@ -100,7 +100,7 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } introduced, ok := versionedPtr.(introducedInterface) - // skip the introduced check for test where currentVersion is 0.0 + // skip the introduced check for test when currentVersion is 0.0 to test all apis if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) diff --git a/pkg/server/genericapiserver.go b/pkg/server/genericapiserver.go index c47fc5b23..7b8e13da2 100644 --- a/pkg/server/genericapiserver.go +++ b/pkg/server/genericapiserver.go @@ -40,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authorization/authorizer" @@ -238,8 +237,6 @@ type GenericAPIServer struct { // StorageVersionManager holds the storage versions of the API resources installed by this server. StorageVersionManager storageversion.Manager - // Version will enable the /version endpoint if non-nil - Version *version.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion diff --git a/pkg/server/genericapiserver_test.go b/pkg/server/genericapiserver_test.go index c2bfef4dd..c6a7cbdcf 100644 --- a/pkg/server/genericapiserver_test.go +++ b/pkg/server/genericapiserver_test.go @@ -138,7 +138,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) { if clientset == nil { t.Fatal("unable to create fake client set") } - + config.EffectiveVersion = utilversion.NewEffectiveVersion("") config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) config.OpenAPIConfig.Info.Version = "unversioned" config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) @@ -460,8 +460,9 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { config.EnableProfiling = true kubeVersion := fakeVersion() - config.Version = &kubeVersion - config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion()) + config.EffectiveVersion = effectiveVersion s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { diff --git a/pkg/server/options/serving_test.go b/pkg/server/options/serving_test.go index 40feeaf62..dc87524ef 100644 --- a/pkg/server/options/serving_test.go +++ b/pkg/server/options/serving_test.go @@ -278,7 +278,6 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) v := fakeVersion() - config.Version = &v config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true diff --git a/pkg/server/storage/resource_encoding_config.go b/pkg/server/storage/resource_encoding_config.go index 5d5cfccc7..7339d17df 100644 --- a/pkg/server/storage/resource_encoding_config.go +++ b/pkg/server/storage/resource_encoding_config.go @@ -157,6 +157,7 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example } // If it was introduced after current compatibility version, don't use it + // skip the introduced check for test when currentVersion is 0.0 to test all apis if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { // API resource lifecycles should be relative to k8s api version majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() diff --git a/pkg/util/version/registry.go b/pkg/util/version/registry.go index 589d42a1a..170ac2804 100644 --- a/pkg/util/version/registry.go +++ b/pkg/util/version/registry.go @@ -59,12 +59,29 @@ var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGloba const ( DefaultKubeComponent = "kube" + + klogLevel = 2 ) +type VersionMapping func(from *version.Version) *version.Version + // ComponentGlobals stores the global variables for a component for easy access. type ComponentGlobals struct { effectiveVersion MutableEffectiveVersion featureGate featuregate.MutableVersionedFeatureGate + + // emulationVersionMapping contains the mapping from the emulation version of this component + // to the emulation version of another component. + emulationVersionMapping map[string]VersionMapping + // dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component. + // If true, the emulation version cannot be set from the flag, or version mapping from another component. + dependentEmulationVersion bool + // minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component + // to the min compatibility version of another component. + minCompatibilityVersionMapping map[string]VersionMapping + // dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component + // If true, the min compatibility version cannot be set from the flag, or version mapping from another component. + dependentMinCompatibilityVersion bool } type ComponentGlobalsRegistry interface { @@ -75,9 +92,8 @@ type ComponentGlobalsRegistry interface { // Returns nil if the component is not registered. FeatureGateFor(component string) featuregate.FeatureGate // Register registers the EffectiveVersion and FeatureGate for a component. - // Overrides existing ComponentGlobals if it is already in the registry if override is true, - // otherwise returns error if the component is already registered. - Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) @@ -85,25 +101,43 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error - // SetAllComponents calls the Validate() function for all the global variables for all components registered. + // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error + // Reset removes all stored ComponentGlobals, configurations, and version mappings. + Reset() + // SetEmulationVersionMapping sets the mapping from the emulation version of one component + // to the emulation version of another component. + // Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent, + // and cannot be set from cmd flags anymore. + // For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed. + SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error } type componentGlobalsRegistry struct { - componentGlobals map[string]ComponentGlobals + componentGlobals map[string]*ComponentGlobals mutex sync.RWMutex - // map of component name to emulation version set from the flag. - emulationVersionConfig cliflag.ConfigurationMap + // list of component name to emulation version set from the flag. + emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string } -func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { +func NewComponentGlobalsRegistry() *componentGlobalsRegistry { return &componentGlobalsRegistry{ - componentGlobals: make(map[string]ComponentGlobals), + componentGlobals: make(map[string]*ComponentGlobals), + emulationVersionConfig: nil, + featureGatesConfig: nil, } } +func (r *componentGlobalsRegistry) Reset() { + r.mutex.RLock() + defer r.mutex.RUnlock() + r.componentGlobals = make(map[string]*ComponentGlobals) + r.emulationVersionConfig = nil + r.featureGatesConfig = nil +} + func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { r.mutex.RLock() defer r.mutex.RUnlock() @@ -124,8 +158,8 @@ func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate. return globals.featureGate } -func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { - if _, ok := r.componentGlobals[component]; ok && !override { +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { + if _, ok := r.componentGlobals[component]; ok { return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { @@ -133,18 +167,23 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return err } } - c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} - r.componentGlobals[component] = c + c := ComponentGlobals{ + effectiveVersion: effectiveVersion, + featureGate: featureGate, + emulationVersionMapping: make(map[string]VersionMapping), + minCompatibilityVersionMapping: make(map[string]VersionMapping), + } + r.componentGlobals[component] = &c return nil } -func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { if effectiveVersion == nil { return fmt.Errorf("cannot register nil effectiveVersion") } r.mutex.Lock() defer r.mutex.Unlock() - return r.unsafeRegister(component, effectiveVersion, featureGate, override) + return r.unsafeRegister(component, effectiveVersion, featureGate) } func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { @@ -154,13 +193,11 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, if ok { return globals.effectiveVersion, globals.featureGate } - utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate)) return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) knownFeatures() []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { var known []string for component, globals := range r.componentGlobals { if globals.featureGate == nil { @@ -174,18 +211,22 @@ func (r *componentGlobalsRegistry) knownFeatures() []string { return known } -func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { var vs []string for component, globals := range r.componentGlobals { binaryVer := globals.effectiveVersion.BinaryVersion() if isEmulation { + if globals.dependentEmulationVersion { + continue + } // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) } else { + if globals.dependentMinCompatibilityVersion { + continue + } // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) @@ -200,51 +241,133 @@ func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { return } r.mutex.Lock() + defer r.mutex.Unlock() for _, globals := range r.componentGlobals { if globals.featureGate != nil { globals.featureGate.Close() } } - r.emulationVersionConfig = make(cliflag.ConfigurationMap) + if r.emulationVersionConfig != nil || r.featureGatesConfig != nil { + klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags") + } + r.emulationVersionConfig = []string{} r.featureGatesConfig = make(map[string][]string) - r.mutex.Unlock() - fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+ "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+ + "If the component is not specified, defaults to \"kube\"") fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ - "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) + "Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n")) +} + +type componentVersion struct { + component string + ver *version.Version +} + +// getFullEmulationVersionConfig expands the given version config with version registered version mapping, +// and returns the map of component to Version. +func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( + versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) { + result := map[string]*version.Version{} + setQueue := []componentVersion{} + for comp, ver := range versionConfigMap { + if _, ok := r.componentGlobals[comp]; !ok { + return result, fmt.Errorf("component not registered: %s", comp) + } + klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String()) + setQueue = append(setQueue, componentVersion{comp, ver}) + } + for len(setQueue) > 0 { + cv := setQueue[0] + if _, visited := result[cv.component]; visited { + return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component) + } + setQueue = setQueue[1:] + result[cv.component] = cv.ver + for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping { + toVer := f(cv.ver) + if toVer == nil { + return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp) + } + klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String()) + setQueue = append(setQueue, componentVersion{toComp, toVer}) + } + } + return result, nil +} + +func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { + m := map[string]*version.Version{} + for _, compVer := range versionConfig { + // default to "kube" of component is not specified + k := "kube" + v := compVer + if strings.Contains(compVer, "=") { + arr := strings.SplitN(compVer, "=", 2) + if len(arr) != 2 { + return m, fmt.Errorf("malformed pair, expect string=string") + } + k = strings.TrimSpace(arr[0]) + v = strings.TrimSpace(arr[1]) + } + ver, err := version.Parse(v) + if err != nil { + return m, err + } + if ver.Patch() != 0 { + return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String()) + } + if existingVer, ok := m[k]; ok { + return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String()) + } + m[k] = ver + } + return m, nil } func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() - for comp, emuVer := range r.emulationVersionConfig { + emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) + if err != nil { + return err + } + for comp := range emulationVersionConfigMap { if _, ok := r.componentGlobals[comp]; !ok { return fmt.Errorf("component not registered: %s", comp) } - klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) - v, err := version.Parse(emuVer) - if err != nil { - return err + // only components without any dependencies can be set from the flag. + if r.componentGlobals[comp].dependentEmulationVersion { + return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp) + } + } + if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil { + return err + } else { + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) } - r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) } // Set feature gate emulation version before setting feature gate flag values. for comp, globals := range r.componentGlobals { if globals.featureGate == nil { continue } - klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) + klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } for comp, fg := range r.featureGatesConfig { if comp == "" { + if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok { + return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use") + } comp = DefaultKubeComponent } if _, ok := r.componentGlobals[comp]; !ok { @@ -255,7 +378,7 @@ func (r *componentGlobalsRegistry) Set() error { return fmt.Errorf("component featureGate not registered: %s", comp) } flagVal := strings.Join(fg, ",") - klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal) if err := featureGate.Set(flagVal); err != nil { return err } @@ -275,3 +398,39 @@ func (r *componentGlobalsRegistry) Validate() []error { } return errs } + +func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error { + if f == nil { + return nil + } + klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent) + r.mutex.Lock() + defer r.mutex.Unlock() + if _, ok := r.componentGlobals[fromComponent]; !ok { + return fmt.Errorf("component not registered: %s", fromComponent) + } + if _, ok := r.componentGlobals[toComponent]; !ok { + return fmt.Errorf("component not registered: %s", toComponent) + } + // check multiple dependency + if r.componentGlobals[toComponent].dependentEmulationVersion { + return fmt.Errorf("mapping of %s already exists from another component", toComponent) + } + r.componentGlobals[toComponent].dependentEmulationVersion = true + + versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping + if _, ok := versionMapping[toComponent]; ok { + return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent) + } + versionMapping[toComponent] = f + klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent) + defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion() + emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) + if err != nil { + return err + } + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) + } + return nil +} diff --git a/pkg/util/version/registry_test.go b/pkg/util/version/registry_test.go index 68625e526..1badd5344 100644 --- a/pkg/util/version/registry_test.go +++ b/pkg/util/version/registry_test.go @@ -22,8 +22,8 @@ import ( "testing" "github.com/spf13/pflag" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" - cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" ) @@ -39,30 +39,23 @@ func TestEffectiveVersionRegistry(t *testing.T) { if r.EffectiveVersionFor(testComponent) != nil { t.Fatalf("expected nil EffectiveVersion initially") } - if err := r.Register(testComponent, ver1, nil, false); err != nil { + if err := r.Register(testComponent, ver1, nil); err != nil { t.Fatalf("expected no error to register new component, but got err: %v", err) } if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version registered") } // overwrite - if err := r.Register(testComponent, ver2, nil, false); err == nil { + if err := r.Register(testComponent, ver2, nil); err == nil { t.Fatalf("expected error to register existing component when override is false") } - if err := r.Register(testComponent, ver2, nil, true); err != nil { - t.Fatalf("expected no error to overriding existing component, but got err: %v", err) - } - if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } func testRegistry(t *testing.T) *componentGlobalsRegistry { - r := componentGlobalsRegistry{ - componentGlobals: map[string]ComponentGlobals{}, - emulationVersionConfig: make(cliflag.ConfigurationMap), - featureGatesConfig: make(map[string][]string), - } + r := NewComponentGlobalsRegistry() verKube := NewEffectiveVersion("1.31") fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ @@ -102,19 +95,35 @@ func testRegistry(t *testing.T) *componentGlobalsRegistry { if err != nil { t.Fatal(err) } - _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) - _ = r.Register(testComponent, verTest, fgTest, true) - return &r + utilruntime.Must(r.Register(DefaultKubeComponent, verKube, fgKube)) + utilruntime.Must(r.Register(testComponent, verTest, fgTest)) + return r } func TestVersionFlagOptions(t *testing.T) { r := testRegistry(t) - emuVers := strings.Join(r.versionFlagOptions(true), "\n") + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" if emuVers != expectedEmuVers { t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) } - minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionFlagOptionsWithMapping(t *testing.T) { + r := testRegistry(t) + utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent, + func(from *version.Version) *version.Version { return from.OffsetMinor(3) })) + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") + expectedEmuVers := "test=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" if minCompVers != expectedMinCompVers { t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) @@ -123,7 +132,7 @@ func TestVersionFlagOptions(t *testing.T) { func TestVersionedFeatureGateFlag(t *testing.T) { r := testRegistry(t) - known := strings.Join(r.knownFeatures(), "\n") + known := strings.Join(r.unsafeKnownFeatures(), "\n") expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + "kube:AllBeta=true|false (BETA - default=false)\n" + "kube:commonC=true|false (BETA - default=true)\n" + @@ -140,86 +149,127 @@ func TestVersionedFeatureGateFlag(t *testing.T) { func TestFlags(t *testing.T) { tests := []struct { name string - emulationVersionFlag string - featureGatesFlag string + flags []string parseError string - expectedKubeEmulationVersion *version.Version - expectedTestEmulationVersion *version.Version + expectedKubeEmulationVersion string + expectedTestEmulationVersion string expectedKubeFeatureValues map[featuregate.Feature]bool expectedTestFeatureValues map[featuregate.Feature]bool }{ { name: "setting kube emulation version", - emulationVersionFlag: "kube=1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + flags: []string{"--emulated-version=kube=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "setting kube emulation version, prefix v ok", - emulationVersionFlag: "kube=v1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + name: "setting kube emulation version twice", + flags: []string{ + "--emulated-version=kube=1.30", + "--emulated-version=kube=1.32", + }, + parseError: "duplicate version flag, kube=1.30 and kube=1.32", + }, + { + name: "prefix v ok", + flags: []string{"--emulated-version=kube=v1.30"}, + expectedKubeEmulationVersion: "1.30", + }, + { + name: "patch version not ok", + flags: []string{"--emulated-version=kube=1.30.2"}, + parseError: "patch version not allowed, got: kube=1.30.2", }, { name: "setting test emulation version", - emulationVersionFlag: "test=2.7", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + flags: []string{"--emulated-version=test=2.7"}, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", }, { - name: "version missing component", - emulationVersionFlag: "1.31", - parseError: "component not registered: 1.31", + name: "version missing component default to kube", + flags: []string{"--emulated-version=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "version unregistered component", - emulationVersionFlag: "test3=1.31", - parseError: "component not registered: test3", + name: "version missing component default to kube with duplicate", + flags: []string{"--emulated-version=1.30", "--emulated-version=kube=1.30"}, + parseError: "duplicate version flag, kube=1.30 and kube=1.30", }, { - name: "invalid version", - emulationVersionFlag: "test=1.foo", - parseError: "illegal version string \"1.foo\"", + name: "version unregistered component", + flags: []string{"--emulated-version=test3=1.31"}, + parseError: "component not registered: test3", }, { - name: "setting test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "invalid version", + flags: []string{"--emulated-version=test=1.foo"}, + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true", + }, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, }, { - name: "setting future test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true,test:testB=true", - parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + name: "setting future test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true,test:testB=true", + }, + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", }, { - name: "setting kube feature flag", - emulationVersionFlag: "test=2.7,kube=1.30", - featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "setting kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + expectedKubeEmulationVersion: "1.30", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, }, { - name: "setting locked kube feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "kubeA=false", - parseError: "cannot set feature gate kubeA to false, feature is locked to true", + name: "setting kube feature flag with different prefix", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kube:kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + parseError: "set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use", }, { - name: "setting unknown test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testD=true", - parseError: "unrecognized feature gate: testD", + name: "setting locked kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=kubeA=false", + }, + parseError: "cannot set feature gate kubeA to false, feature is locked to true", }, { - name: "setting unknown component feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test3:commonC=true", - parseError: "component not registered: test3", + name: "setting unknown test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testD=true", + }, + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test3:commonC=true", + }, + parseError: "component not registered: test3", }, } for i, test := range tests { @@ -227,9 +277,7 @@ func TestFlags(t *testing.T) { fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) r := testRegistry(t) r.AddFlags(fs) - - err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), - fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + err := fs.Parse(test.flags) if err == nil { err = r.Set() } @@ -242,19 +290,11 @@ func TestFlags(t *testing.T) { if err != nil { t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) } - if test.expectedKubeEmulationVersion != nil { - v := r.EffectiveVersionFor("kube").EmulationVersion() - if !v.EqualTo(test.expectedKubeEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) - return - } + if len(test.expectedKubeEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).EmulationVersion(), test.expectedKubeEmulationVersion) } - if test.expectedTestEmulationVersion != nil { - v := r.EffectiveVersionFor("test").EmulationVersion() - if !v.EqualTo(test.expectedTestEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) - return - } + if len(test.expectedTestEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion) } for f, v := range test.expectedKubeFeatureValues { if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { @@ -269,3 +309,110 @@ func TestFlags(t *testing.T) { }) } } + +func TestVersionMapping(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11") + + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r.AddFlags(fs) + + if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil { + t.Fatal(err) + return + } + if err := r.Set(); err != nil { + t.Fatal(err) + return + } + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09") +} + +func TestVersionMappingWithMultipleDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + err := r.SetEmulationVersionMapping("test3", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-1, from.Minor()+19) + }) + if err == nil { + t.Errorf("expect error when setting 2nd mapping to test2") + } +} + +func TestVersionMappingWithCyclicDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + err := r.SetEmulationVersionMapping("test3", "test1", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-2, from.Minor()+48) + }) + if err == nil { + t.Errorf("expect cyclic version mapping error") + } +} + +func assertVersionEqualTo(t *testing.T, ver *version.Version, expectedVer string) { + if ver.EqualTo(version.MustParse(expectedVer)) { + return + } + t.Errorf("expected: %s, got %s", expectedVer, ver.String()) +} diff --git a/pkg/util/version/version.go b/pkg/util/version/version.go index fce927c6c..1596aef38 100644 --- a/pkg/util/version/version.go +++ b/pkg/util/version/version.go @@ -18,10 +18,8 @@ package version import ( "fmt" - "strings" "sync/atomic" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" baseversion "k8s.io/component-base/version" ) @@ -40,37 +38,6 @@ type MutableEffectiveVersion interface { Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) SetEmulationVersion(emulationVersion *version.Version) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) - // AddFlags adds the "{prefix}-emulated-version" to the flagset. - AddFlags(fs *pflag.FlagSet, prefix string) -} - -type VersionVar struct { - val atomic.Pointer[version.Version] -} - -// Set sets the flag value -func (v *VersionVar) Set(s string) error { - components := strings.Split(s, ".") - if len(components) != 2 { - return fmt.Errorf("version %s is not in the format of major.minor", s) - } - ver, err := version.ParseGeneric(s) - if err != nil { - return err - } - v.val.Store(ver) - return nil -} - -// String returns the flag value -func (v *VersionVar) String() string { - ver := v.val.Load() - return ver.String() -} - -// Type gets the flag type -func (v *VersionVar) Type() string { - return "version" } type effectiveVersion struct { @@ -78,9 +45,9 @@ type effectiveVersion struct { // If the emulationVersion is set by the users, it could only contain major and minor versions. // In tests, emulationVersion could be the same as the binary version, or set directly, // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - emulationVersion VersionVar + emulationVersion atomic.Pointer[version.Version] // minCompatibilityVersion could only contain major and minor versions. - minCompatibilityVersion VersionVar + minCompatibilityVersion atomic.Pointer[version.Version] } func (m *effectiveVersion) BinaryVersion() *version.Version { @@ -88,13 +55,17 @@ func (m *effectiveVersion) BinaryVersion() *version.Version { } func (m *effectiveVersion) EmulationVersion() *version.Version { - // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - // The pre-release should not be accessible to the users. - return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) + ver := m.emulationVersion.Load() + if ver != nil { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return ver.WithPreRelease(m.BinaryVersion().PreRelease()) + } + return ver } func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { - return m.minCompatibilityVersion.val.Load() + return m.minCompatibilityVersion.Load() } func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { @@ -109,26 +80,33 @@ func (m *effectiveVersion) String() string { m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) } +func majorMinor(ver *version.Version) *version.Version { + if ver == nil { + return ver + } + return version.MajorMinor(ver.Major(), ver.Minor()) +} + func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { m.binaryVersion.Store(binaryVersion) - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) } func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) Validate() []error { var errs []error // Validate only checks the major and minor versions. binaryVersion := m.binaryVersion.Load().WithPatch(0) - emulationVersion := m.emulationVersion.val.Load() - minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + emulationVersion := m.emulationVersion.Load() + minCompatibilityVersion := m.minCompatibilityVersion.Load() // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion @@ -151,45 +129,36 @@ func (m *effectiveVersion) Validate() []error { return errs } -// AddFlags adds the "{prefix}-emulated-version" to the flagset. -func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { - if m == nil { - return - } - if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { - prefix += "-" - } - fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ - "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ - "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ - "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ - "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ - "Format could only be major.minor") -} - -func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { +func newEffectiveVersion(binaryVersion *version.Version) MutableEffectiveVersion { effective := &effectiveVersion{} - binaryVersion := version.MustParse(binaryVer) compatVersion := binaryVersion.SubtractMinor(1) effective.Set(binaryVersion, binaryVersion, compatVersion) return effective } +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + if binaryVer == "" { + return &effectiveVersion{} + } + binaryVersion := version.MustParse(binaryVer) + return newEffectiveVersion(binaryVersion) +} + // DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the // current build information. func DefaultBuildEffectiveVersion() MutableEffectiveVersion { verInfo := baseversion.Get() - ver := NewEffectiveVersion(verInfo.String()) - if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { - ver = DefaultKubeEffectiveVersion() + binaryVersion := version.MustParse(verInfo.String()).WithInfo(verInfo) + if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 { + return DefaultKubeEffectiveVersion() } - return ver + return newEffectiveVersion(binaryVersion) } // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. // Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - return NewEffectiveVersion("1.31") + binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + return newEffectiveVersion(binaryVersion) } diff --git a/pkg/util/version/version_test.go b/pkg/util/version/version_test.go index 24db0318f..784e3e4e1 100644 --- a/pkg/util/version/version_test.go +++ b/pkg/util/version/version_test.go @@ -17,11 +17,8 @@ limitations under the License. package version import ( - "fmt" - "strings" "testing" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" ) @@ -127,54 +124,3 @@ func TestValidate(t *testing.T) { }) } } - -func TestEffectiveVersionsFlag(t *testing.T) { - tests := []struct { - name string - emulationVersion string - expectedEmulationVersion *version.Version - parseError string - }{ - { - name: "major.minor ok", - emulationVersion: "1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "v prefix ok", - emulationVersion: "v1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "semantic version not ok", - emulationVersion: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", - }, - { - name: "invalid version", - emulationVersion: "1.foo", - parseError: "illegal version string", - }, - } - for i, test := range tests { - t.Run(test.name, func(t *testing.T) { - fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) - effective := NewEffectiveVersion("1.30") - effective.AddFlags(fs, "test") - - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) - if test.parseError != "" { - if !strings.Contains(err.Error(), test.parseError) { - t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) - } - return - } - if err != nil { - t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) - } - if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { - t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) - } - }) - } -} From b26fefe178bca0fd72307048a786f62c989b2e49 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 10 Jun 2024 17:50:22 +0000 Subject: [PATCH 4/4] add DefaultComponentGlobalsRegistry flags in ServerRunOptions Signed-off-by: Siyuan Zhang Kubernetes-commit: 379676c4bef48e5d2add28851302b55b41fcabcf --- pkg/server/config.go | 8 +-- pkg/server/config_test.go | 2 + pkg/server/genericapiserver.go | 9 ++- pkg/server/options/api_enablement_test.go | 1 - pkg/server/options/server_run_options.go | 50 +++++++------- pkg/server/options/server_run_options_test.go | 66 +++++++++++-------- .../storage/resource_encoding_config.go | 6 +- pkg/util/version/registry.go | 18 +++++ pkg/util/version/version.go | 9 +-- pkg/util/version/version_test.go | 19 ------ 10 files changed, 98 insertions(+), 90 deletions(-) diff --git a/pkg/server/config.go b/pkg/server/config.go index 931226bb6..6f0ca1bca 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -832,7 +832,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { manager := c.AggregatedDiscoveryGroupManager if manager == nil { manager = discoveryendpoint.NewResourceManager("apis") @@ -1047,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled()) } handler = genericfilters.WithHTTPLogging(handler) - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { + if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) { handler = genericapifilters.WithTracing(handler, c.TracerProvider) } handler = genericapifilters.WithLatencyTrackers(handler) // WithRoutine will execute future handlers in a separate goroutine and serving // handler in current goroutine to minimize the stack memory usage. It must be // after WithPanicRecover() to be protected from panics. - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { + if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { handler = genericfilters.WithRoutine(handler, c.LongRunningFunc) } handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) @@ -1098,7 +1098,7 @@ func installAPI(s *GenericAPIServer, c *Config) { routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{})) } else { diff --git a/pkg/server/config_test.go b/pkg/server/config_test.go index 6d25272b0..a7f76d014 100644 --- a/pkg/server/config_test.go +++ b/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -308,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) { LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false }, lifecycleSignals: newLifecycleSignals(), TracerProvider: tracing.NewNoopTracerProvider(), + FeatureGate: utilfeature.DefaultFeatureGate, } h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/server/genericapiserver.go b/pkg/server/genericapiserver.go index 7b8e13da2..fed94bf4a 100644 --- a/pkg/server/genericapiserver.go +++ b/pkg/server/genericapiserver.go @@ -51,7 +51,6 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" "k8s.io/component-base/featuregate" @@ -780,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A } resourceInfos = append(resourceInfos, r...) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { // Aggregated discovery only aggregates resources under /apis if apiPrefix == APIGroupPrefix { s.AggregatedDiscoveryGroupManager.AddGroupVersion( @@ -808,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A s.RegisterDestroyFunc(apiGroupInfo.destroyStorage) - if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) { + if s.FeatureGate.Enabled(features.StorageVersionAPI) && + s.FeatureGate.Enabled(features.APIServerIdentity) { // API installation happens before we start listening on the handlers, // therefore it is safe to register ResourceInfos here. The handler will block // write requests until the storage versions of the targeting resources are updated. @@ -839,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo // Install the version handler. // Add a handler at / to enumerate the supported api versions. legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{})) } else { diff --git a/pkg/server/options/api_enablement_test.go b/pkg/server/options/api_enablement_test.go index 7d6024400..a14319e53 100644 --- a/pkg/server/options/api_enablement_test.go +++ b/pkg/server/options/api_enablement_test.go @@ -33,7 +33,6 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { name string - testOptions *APIEnablementOptions runtimeConfig cliflag.ConfigurationMap expectErr string }{ diff --git a/pkg/server/options/server_run_options.go b/pkg/server/options/server_run_options.go index 88dc3802f..593d59163 100644 --- a/pkg/server/options/server_run_options.go +++ b/pkg/server/options/server_run_options.go @@ -25,9 +25,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -91,12 +92,23 @@ type ServerRunOptions struct { // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration - // FeatureGate are the featuregate to install on the CLI - FeatureGate featuregate.FeatureGate - EffectiveVersion utilversion.EffectiveVersion + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry + // ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry. + ComponentName string } -func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { +func NewServerRunOptions() *ServerRunOptions { + if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil { + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) + } + + return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry) +} + +func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -109,13 +121,16 @@ func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion u JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentName: componentName, + ComponentGlobalsRegistry: componentGlobalsRegistry, } } // ApplyTo applies the run options to the method receiver and returns self func (s *ServerRunOptions) ApplyTo(c *server.Config) error { + if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil { + return err + } c.CorsAllowedOriginList = s.CorsAllowedOriginList c.HSTSDirectives = s.HSTSDirectives c.ExternalAddress = s.ExternalHost @@ -131,8 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod - c.EffectiveVersion = s.EffectiveVersion - c.FeatureGate = s.FeatureGate + c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName) + c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName) return nil } @@ -205,12 +220,7 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } - if s.FeatureGate != nil { - if errs := s.FeatureGate.Validate(); len(errs) != 0 { - errors = append(errors, errs...) - } - } - if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 { errors = append(errors, errs...) } return errors @@ -353,15 +363,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") + + s.ComponentGlobalsRegistry.AddFlags(fs) } // Complete fills missing fields with defaults. func (s *ServerRunOptions) Complete() error { - if s.FeatureGate == nil { - return fmt.Errorf("nil FeatureGate in ServerRunOptions") - } - if s.EffectiveVersion == nil { - return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") - } - return nil + return s.ComponentGlobalsRegistry.SetFallback() } diff --git a/pkg/server/options/server_run_options_test.go b/pkg/server/options/server_run_options_test.go index 0c62a4d68..7bfcf0245 100644 --- a/pkg/server/options/server_run_options_test.go +++ b/pkg/server/options/server_run_options_test.go @@ -23,14 +23,21 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + testRegistry := utilversion.NewComponentGlobalsRegistry() featureGate := utilfeature.DefaultFeatureGate.DeepCopy() effectiveVersion := utilversion.NewEffectiveVersion("1.30") + effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32)) + testComponent := "test" + utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate)) + testCases := []struct { name string testOptions *ServerRunOptions @@ -47,8 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -63,8 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -79,8 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--request-timeout can not be negative value", }, @@ -95,8 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -111,8 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -127,8 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -144,8 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -161,8 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -178,11 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, + { + name: "Test when emulation version is invalid", + testOptions: &ServerRunOptions{ + AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), + CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"}, + HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"}, + MaxRequestsInFlight: 400, + MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, + MinRequestTimeout: 1800, + JSONPatchMaxCopyBytes: 10 * 1024 * 1024, + MaxRequestBodyBytes: 10 * 1024 * 1024, + ComponentName: testComponent, + ComponentGlobalsRegistry: testRegistry, + }, + expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]", + }, { name: "Test when ServerRunOptions is valid", testOptions: &ServerRunOptions{ @@ -195,8 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, }, } @@ -216,8 +230,6 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -265,7 +277,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -289,8 +301,6 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -299,13 +309,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions(featureGate, effectiveVersion) + return NewServerRunOptions() }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions(featureGate, effectiveVersion) + o := NewServerRunOptions() o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -332,7 +342,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/pkg/server/storage/resource_encoding_config.go b/pkg/server/storage/resource_encoding_config.go index 7339d17df..d73c8e62c 100644 --- a/pkg/server/storage/resource_encoding_config.go +++ b/pkg/server/storage/resource_encoding_config.go @@ -181,9 +181,9 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example // was being served which didn't exist at all in min-compatibility-version. // // In the alpha case - we do not support compatibility versioning of - // alpha types and recommend users do not mix the two. + // alpha types and recommend users do not mix the two. // In the skip-level case - The version of apiserver we are retaining - // compatibility with has no knowledge of the type, - // so storing it in another type is no issue. + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. return binaryVersionOfResource, nil } diff --git a/pkg/util/version/registry.go b/pkg/util/version/registry.go index 170ac2804..e6655277c 100644 --- a/pkg/util/version/registry.go +++ b/pkg/util/version/registry.go @@ -101,6 +101,8 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error + // SetFallback calls Set() if it has never been called. + SetFallback() error // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error // Reset removes all stored ComponentGlobals, configurations, and version mappings. @@ -120,6 +122,8 @@ type componentGlobalsRegistry struct { emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string + // set stores if the Set() function for the registry is already called. + set bool } func NewComponentGlobalsRegistry() *componentGlobalsRegistry { @@ -136,6 +140,7 @@ func (r *componentGlobalsRegistry) Reset() { r.componentGlobals = make(map[string]*ComponentGlobals) r.emulationVersionConfig = nil r.featureGatesConfig = nil + r.set = false } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -330,9 +335,22 @@ func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { return m, nil } +func (r *componentGlobalsRegistry) SetFallback() error { + r.mutex.Lock() + set := r.set + r.mutex.Unlock() + if set { + return nil + } + klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" + + " right after parsing flags to avoid using feature gates before their final values are set by the flags.") + return r.Set() +} + func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() + r.set = true emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) if err != nil { return err diff --git a/pkg/util/version/version.go b/pkg/util/version/version.go index 1596aef38..a7a5fda87 100644 --- a/pkg/util/version/version.go +++ b/pkg/util/version/version.go @@ -111,12 +111,6 @@ func (m *effectiveVersion) Validate() []error { // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion minEmuVer := binaryVersion.SubtractMinor(1) - // TODO: remove in 1.32 - // emulationVersion is introduced in 1.31, so it cannot be lower than that. - // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. - if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { - minEmuVer = version.MajorMinor(1, 31) - } if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) } @@ -157,8 +151,7 @@ func DefaultBuildEffectiveVersion() MutableEffectiveVersion { // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. -// Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get()) return newEffectiveVersion(binaryVersion) } diff --git a/pkg/util/version/version_test.go b/pkg/util/version/version_test.go index 784e3e4e1..aff8a8e4a 100644 --- a/pkg/util/version/version_test.go +++ b/pkg/util/version/version_test.go @@ -42,25 +42,6 @@ func TestValidate(t *testing.T) { emulationVersion: "v1.31.0", minCompatibilityVersion: "v1.31.0", }, - { - name: "binary version 1.31, emulation version lower than 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.30.0", - minCompatibilityVersion: "v1.30.0", - expectErrors: true, - }, - { - name: "binary version 1.31, emulation version 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.30.0", - }, - { - name: "binary version lower than 1.31", - binaryVersion: "v1.30.2", - emulationVersion: "v1.29.0", - minCompatibilityVersion: "v1.29.0", - }, { name: "emulation version two minor lower than binary not ok", binaryVersion: "v1.33.2",