kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #122891 from siyuanfoundation/api-comp-ver1 

apimachinery: API Emulation Versioning

Kubernetes-commit: 7a6062f4c13d0b7407876b325ce52c43de18f92f
This commit is contained in:
Kubernetes Publisher 2024-06-25 20:04:48 -07:00
parent 7920da295d b26fefe178
commit 348f8e3cff
26 changed files with 1822 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -45,9 +45,9 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/square/go-jose.v2 v2.6.0
k8s.io/api v0.0.0-20240626062052-149781fc54f5
k8s.io/apimachinery v0.0.0-20240626061445-a05248b07a6e
k8s.io/apimachinery v0.0.0-20240626061446-c225984b7bed
k8s.io/client-go v0.0.0-20240626062855-8ffa5314741e
k8s.io/component-base v0.0.0-20240626064639-1f2e30104e8a
k8s.io/component-base v0.0.0-20240626064641-0eb10f703efe
k8s.io/klog/v2 v2.130.1
k8s.io/kms v0.0.0-20240626065322-b47e46c9b25f
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340

8
go.sum
View File

@ -372,12 +372,12 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.0.0-20240626062052-149781fc54f5 h1:yqS/8fRAivOgCF1sgfIwxp1A/S2VrPpudN0V3RfHj6M=
k8s.io/api v0.0.0-20240626062052-149781fc54f5/go.mod h1:WzXtjaoUCXrzbvJtK/lCWCih7nCdOFqgkgptCo1OH/w=
k8s.io/apimachinery v0.0.0-20240626061445-a05248b07a6e h1:DBEwrTm4pQoqxKDZM1fY9oZ4ncSkHlmnKDFbu/icsHE=
k8s.io/apimachinery v0.0.0-20240626061445-a05248b07a6e/go.mod h1:WJc1RfanAukQew7I55uKC34w5zx50UFDD5qo/JD4dNE=
k8s.io/apimachinery v0.0.0-20240626061446-c225984b7bed h1:uJ7kyuzfVNEOMtzKLgbR4aUBZ1a++mNQPgBOeNii610=
k8s.io/apimachinery v0.0.0-20240626061446-c225984b7bed/go.mod h1:WJc1RfanAukQew7I55uKC34w5zx50UFDD5qo/JD4dNE=
k8s.io/client-go v0.0.0-20240626062855-8ffa5314741e h1:xKbYBQXJjUyvJ7lmp5bJz0Ufnoa/3Ybr4GZ42Mf9COM=
k8s.io/client-go v0.0.0-20240626062855-8ffa5314741e/go.mod h1:10qMvliiExgeiO4z62WzFt6kQk3F4UlvFyXpjK5e8FA=
k8s.io/component-base v0.0.0-20240626064639-1f2e30104e8a h1:6Vf1BT3V25kiYJaFOhhTkQyEHXdepiEYWRgPdh+i2AU=
k8s.io/component-base v0.0.0-20240626064639-1f2e30104e8a/go.mod h1:HLRj6WUqGEeDExNNYzOZUHafJvSpSzPyxZJEhBgiqjI=
k8s.io/component-base v0.0.0-20240626064641-0eb10f703efe h1:eXzjveqpDKwB8NPy8lyEAzs/7f/xb8Gk/6DCDQ02/0w=
k8s.io/component-base v0.0.0-20240626064641-0eb10f703efe/go.mod h1:lAq7g1d6gPIb29qbxU/iqz34CASTgVPP0pB1x1/2GAo=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kms v0.0.0-20240626065322-b47e46c9b25f h1:rGbqzko9e+3Ubbn3wxJCv1dLBZpXksiINmPlb1/5NsY=

21
pkg/cel/environment/base.go
View File

@ -30,20 +30,27 @@ import (
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/cel/library"
utilversion "k8s.io/apiserver/pkg/util/version"
)
// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
// that guarantees compatibility with CEL features/libraries/parameters understood by
// an n-1 version
// the api server min compatibility version
//
// This default will be set to no more than n-1 the current Kubernetes major.minor version.
// This default will be set to no more than the current Kubernetes major.minor version.
//
// Note that a default version number less than n-1 indicates a wider range of version
// compatibility than strictly required for rollback. A wide range of compatibility is
// desirable because it means that CEL expressions are portable across a wider range
// of Kubernetes versions.
// Note that a default version number less than n-1 the current Kubernetes major.minor version
// indicates a wider range of version compatibility than strictly required for rollback.
// A wide range of compatibility is desirable because it means that CEL expressions are portable
// across a wider range of Kubernetes versions.
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
return version.MajorMinor(1, 30)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}
return effectiveVer.MinCompatibilityVersion()
}
var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt)

11
pkg/features/kube_features.go
View File

@ -322,6 +322,17 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
}
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}
// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

8
pkg/registry/generic/options.go
View File

@ -19,6 +19,7 @@ package generic
import (
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@ -39,12 +40,15 @@ type RESTOptions struct {
}
// Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests)
func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) {
func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) {
return opts, nil
}
type RESTOptionsGetter interface {
GetRESTOptions(resource schema.GroupResource) (RESTOptions, error)
// GetRESTOptions returns the RESTOptions for the given resource and example object.
// The example object is used to determine the storage version for the resource.
// If the example object is nil, the storage version will be determined by the resource's default storage version.
GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error)
}
// StoreOptions is set of configuration options used to complete generic registries.

2
pkg/registry/generic/registry/store.go
View File

@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
return err
}
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc())
if err != nil {
return err
}

38
pkg/server/config.go
View File

@ -42,8 +42,8 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/authenticator"
@ -70,8 +70,10 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
@ -147,8 +149,11 @@ type Config struct {
// done values in this values for this map are ignored.
PostStartHooks map[string]PostStartHookConfigEntry
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// AuditBackend is where audit events are sent to.
AuditBackend audit.Backend
// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
@ -585,7 +590,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
}
}
func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
if config == nil {
return
}
@ -624,7 +629,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
}
}
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
if config == nil {
return
}
@ -676,6 +681,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
// Complete fills in any fields not set that are required to have valid data and can be derived
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
if c.FeatureGate == nil {
c.FeatureGate = utilfeature.DefaultFeatureGate
}
if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
c.ExternalAddress = c.PublicAddress.String()
}
@ -691,9 +699,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
}
c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
}
completeOpenAPI(c.OpenAPIConfig, c.Version)
completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion())
completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion())
if c.DiscoveryAddresses == nil {
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@ -711,7 +718,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
} else {
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
// use the storage prefix as the key if possible
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil {
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
return opts.ResourcePrefix
}
// otherwise return "" to use the default key (parent GV name)
@ -819,12 +826,13 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
APIServerID: c.APIServerID,
StorageVersionManager: c.StorageVersionManager,
Version: c.Version,
EffectiveVersion: c.EffectiveVersion,
FeatureGate: c.FeatureGate,
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
manager := c.AggregatedDiscoveryGroupManager
if manager == nil {
manager = discoveryendpoint.NewResourceManager("apis")
@ -1039,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
}
handler = genericfilters.WithHTTPLogging(handler)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
handler = genericapifilters.WithTracing(handler, c.TracerProvider)
}
handler = genericapifilters.WithLatencyTrackers(handler)
// WithRoutine will execute future handlers in a separate goroutine and serving
// handler in current goroutine to minimize the stack memory usage. It must be
// after WithPanicRecover() to be protected from panics.
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
handler = genericfilters.WithRoutine(handler, c.LongRunningFunc)
}
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
@ -1087,10 +1095,10 @@ func installAPI(s *GenericAPIServer, c *Config) {
}
}
routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer)
routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
if c.EnableDiscovery {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
} else {

5
pkg/server/config_test.go
View File

@ -40,6 +40,8 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/healthz"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
@ -90,6 +92,7 @@ func TestNewWithDelegate(t *testing.T) {
delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
delegateConfig.LoopbackClientConfig = &rest.Config{}
delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
clientset := fake.NewSimpleClientset()
if clientset == nil {
t.Fatal("unable to create fake client set")
@ -122,6 +125,7 @@ func TestNewWithDelegate(t *testing.T) {
wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
wrappingConfig.LoopbackClientConfig = &rest.Config{}
wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error {
return fmt.Errorf("wrapping failed healthcheck")
@ -305,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) {
LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false },
lifecycleSignals: newLifecycleSignals(),
TracerProvider: tracing.NewNoopTracerProvider(),
FeatureGate: utilfeature.DefaultFeatureGate,
}
h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

65
pkg/server/deleted_kinds.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package server
import (
"fmt"
"os"
"strconv"
"strings"
@ -25,16 +26,15 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/klog/v2"
)
// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
type resourceExpirationEvaluator struct {
currentMajor int
currentMinor int
isAlpha bool
currentVersion *apimachineryversion.Version
isAlpha bool
// This is usually set for testing for which tests need to be removed. This prevent insta-failing CI.
// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
strictRemovedHandlingInAlpha bool
@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface {
ShouldServeForVersion(majorRemoved, minorRemoved int) bool
}
func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
if currentVersion == nil {
return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
}
klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
ret := &resourceExpirationEvaluator{
strictRemovedHandlingInAlpha: false,
}
if len(currentVersion.Major) > 0 {
currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
if err != nil {
return nil, err
}
ret.currentMajor = int(currentMajor64)
}
if len(currentVersion.Minor) > 0 {
// split the "normal" + and - for semver stuff
minorString := strings.Split(currentVersion.Minor, "+")[0]
minorString = strings.Split(minorString, "-")[0]
minorString = strings.Split(minorString, ".")[0]
currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
if err != nil {
return nil, err
}
ret.currentMinor = int(currentMinor64)
}
ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
// Only keeps the major and minor versions from input version.
ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
// do nothing
@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
return false
}
introduced, ok := versionedPtr.(introducedInterface)
// skip the introduced check for test when currentVersion is 0.0 to test all apis
if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) {
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if e.currentVersion.LessThan(verIntroduced) {
return false
}
}
removed, ok := versionedPtr.(removedInterface)
if !ok {
return true
@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
}
func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
if e.currentMajor < majorRemoved {
removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
if removedVer.GreaterThan(e.currentVersion) {
return true
}
if e.currentMajor > majorRemoved {
return false
}
if e.currentMinor < minorRemoved {
return true
}
if e.currentMinor > minorRemoved {
if removedVer.LessThan(e.currentVersion) {
return false
}
// at this point major and minor are equal, so this API should be removed when the current release GAs.
@ -152,6 +144,11 @@ type removedInterface interface {
APILifecycleRemoved() (major, minor int)
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
}
klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
storage := versionToResource[resourceName]
storage.Destroy()
delete(versionToResource, resourceName)
}
versionedResourcesStorageMap[apiVersion] = versionToResource

173
pkg/server/deleted_kinds_test.go
View File

@ -25,57 +25,41 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/dump"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
)
func Test_newResourceExpirationEvaluator(t *testing.T) {
tests := []struct {
name string
currentVersion version.Info
currentVersion string
expected resourceExpirationEvaluator
expectedErr string
}{
{
name: "beta",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "beta",
currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "alpha",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true},
name: "alpha",
currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true},
},
{
name: "maintenance",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.1",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "maintenance",
currentVersion: "v1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "bad",
currentVersion: version.Info{
Major: "1",
Minor: "20something+",
GitVersion: "v1.20.1",
},
expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`,
name: "no v prefix",
currentVersion: "1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion)
actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion))
checkErr(t, actualErr, tt.expectedErr)
if actualErr != nil {
@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) {
}
}
func storageRemovedIn(major, minor int) removedInStorage {
return removedInStorage{major: major, minor: minor}
func storageRemovedIn(major, minor int) *removedInStorage {
return &removedInStorage{major: major, minor: minor}
}
func storageNeverRemoved() removedInStorage {
return removedInStorage{neverRemoved: true}
func storageNeverRemoved() *removedInStorage {
return &removedInStorage{neverRemoved: true}
}
type removedInStorage struct {
@ -103,23 +87,23 @@ type removedInStorage struct {
neverRemoved bool
}
func (r removedInStorage) New() runtime.Object {
func (r *removedInStorage) New() runtime.Object {
if r.neverRemoved {
return neverRemovedObj{}
return &defaultObj{}
}
return removedInObj{major: r.major, minor: r.minor}
return &removedInObj{major: r.major, minor: r.minor}
}
func (r removedInStorage) Destroy() {
func (r *removedInStorage) Destroy() {
}
type neverRemovedObj struct {
type defaultObj struct {
}
func (r neverRemovedObj) GetObjectKind() schema.ObjectKind {
func (r *defaultObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r neverRemovedObj) DeepCopyObject() runtime.Object {
func (r *defaultObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
@ -127,13 +111,45 @@ type removedInObj struct {
major, minor int
}
func (r removedInObj) GetObjectKind() schema.ObjectKind {
func (r *removedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r removedInObj) DeepCopyObject() runtime.Object {
func (r *removedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r removedInObj) APILifecycleRemoved() (major, minor int) {
func (r *removedInObj) APILifecycleRemoved() (major, minor int) {
return r.major, r.minor
}
func storageIntroducedIn(major, minor int) *introducedInStorage {
return &introducedInStorage{major: major, minor: minor}
}
type introducedInStorage struct {
major, minor int
}
func (r *introducedInStorage) New() runtime.Object {
if r.major == 0 && r.minor == 0 {
return &defaultObj{}
}
return &IntroducedInObj{major: r.major, minor: r.minor}
}
func (r *introducedInStorage) Destroy() {
}
type IntroducedInObj struct {
major, minor int
}
func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r *IntroducedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) {
return r.major, r.minor
}
@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 20),
expected: false,
@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-deferred",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
isAlpha: true,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
},
restStorage: storageRemovedIn(1, 20),
expected: true,
@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha-but-strict",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
strictRemovedHandlingInAlpha: true,
},
@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-deferral-does-not-help",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 21,
currentVersion: apimachineryversion.MajorMinor(1, 21),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 2,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(2, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 21),
expected: true,
@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "never-removed",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageNeverRemoved(),
expected: true,
},
{
name: "introduced-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 20),
expected: true,
},
{
name: "introduced-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 19),
expected: true,
},
{
name: "introduced-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 21),
expected: false,
},
{
name: "missing-introduced",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(0, 0),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-one-of-two",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-nested-not-expired",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-all-of-version",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {

19
pkg/server/genericapiserver.go
View File

@ -40,7 +40,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
@ -52,8 +51,9 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
openapicommon "k8s.io/kube-openapi/pkg/common"
@ -236,8 +236,11 @@ type GenericAPIServer struct {
// StorageVersionManager holds the storage versions of the API resources installed by this server.
StorageVersionManager storageversion.Manager
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
lifecycleSignals lifecycleSignals
@ -776,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
}
resourceInfos = append(resourceInfos, r...)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
// Aggregated discovery only aggregates resources under /apis
if apiPrefix == APIGroupPrefix {
s.AggregatedDiscoveryGroupManager.AddGroupVersion(
@ -804,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
s.FeatureGate.Enabled(features.APIServerIdentity) {
// API installation happens before we start listening on the handlers,
// therefore it is safe to register ResourceInfos here. The handler will block
// write requests until the storage versions of the targeting resources are updated.
@ -835,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
// Install the version handler.
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
} else {

9
pkg/server/genericapiserver_test.go
View File

@ -48,6 +48,7 @@ import (
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
"k8s.io/apiserver/pkg/registry/rest"
genericfilters "k8s.io/apiserver/pkg/server/filters"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
@ -137,7 +138,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) {
if clientset == nil {
t.Fatal("unable to create fake client set")
}
config.EffectiveVersion = utilversion.NewEffectiveVersion("")
config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
config.OpenAPIConfig.Info.Version = "unversioned"
config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
@ -459,7 +460,9 @@ func TestNotRestRoutesHaveAuth(t *testing.T) {
config.EnableProfiling = true
kubeVersion := fakeVersion()
config.Version = &kubeVersion
effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String())
effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion())
config.EffectiveVersion = effectiveVersion
s, err := config.Complete(nil).New("test", NewEmptyDelegate())
if err != nil {
@ -586,7 +589,7 @@ func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitVersion: "42.42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
BuildDate: time.Now().String(),

41
pkg/server/options/api_enablement_test.go
View File

@ -32,46 +32,41 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
func TestAPIEnablementOptionsValidate(t *testing.T) {
testCases := []struct {
name string
testOptions *APIEnablementOptions
expectErr string
name string
runtimeConfig cliflag.ConfigurationMap
expectErr string
}{
{
name: "test when options is nil",
},
{
name: "test when invalid key with only api/all=false",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
},
expectErr: "invalid key with only api/all=false",
name: "test when invalid key with only api/all=false",
runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
expectErr: "invalid key with only api/all=false",
},
{
name: "test when ConfigurationMap key is invalid",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
},
expectErr: "runtime-config invalid key",
name: "test when ConfigurationMap key is invalid",
runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
expectErr: "runtime-config invalid key",
},
{
name: "test when unknown api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
},
expectErr: "unknown api groups",
name: "test when unknown api groups",
runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
expectErr: "unknown api groups",
},
{
name: "test when valid api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
name: "test when valid api groups",
runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
}
testGroupRegistry := fakeGroupRegistry{}
for _, testcase := range testCases {
t.Run(testcase.name, func(t *testing.T) {
errs := testcase.testOptions.Validate(testGroupRegistry)
testOptions := &APIEnablementOptions{
RuntimeConfig: testcase.runtimeConfig,
}
errs := testOptions.Validate(testGroupRegistry)
if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) {
t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr)
}

10
pkg/server/options/etcd.go
View File

@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
StorageFactory serverstorage.StorageFactory
}
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource)
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
}
@ -469,7 +469,7 @@ type SimpleStorageFactory struct {
StorageConfig storagebackend.Config
}
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return s.StorageConfig.ForResource(resource), nil
}
@ -493,8 +493,8 @@ type transformerStorageFactory struct {
resourceTransformers storagevalue.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource)
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource, example)
if err != nil {
return nil, err
}

2
pkg/server/options/etcd_test.go
View File

@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) {
if err := etcdOptions.ApplyTo(serverConfig); err != nil {
t.Fatalf("Failed to apply etcd options error: %v", err)
}
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

34
pkg/server/options/server_run_options.go
View File

@ -25,8 +25,10 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"github.com/spf13/pflag"
)
@ -89,9 +91,24 @@ type ServerRunOptions struct {
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
ComponentName string
}
func NewServerRunOptions() *ServerRunOptions {
if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
}
return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
}
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
@ -104,11 +121,16 @@ func NewServerRunOptions() *ServerRunOptions {
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
ComponentName: componentName,
ComponentGlobalsRegistry: componentGlobalsRegistry,
}
}
// ApplyTo applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
return err
}
c.CorsAllowedOriginList = s.CorsAllowedOriginList
c.HSTSDirectives = s.HSTSDirectives
c.ExternalAddress = s.ExternalHost
@ -124,6 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
return nil
}
@ -196,6 +220,9 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
return errors
}
@ -337,5 +364,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
s.ComponentGlobalsRegistry.AddFlags(fs)
}
// Complete fills missing fields with defaults.
func (s *ServerRunOptions) Complete() error {
return s.ComponentGlobalsRegistry.SetFallback()
}

38
pkg/server/options/server_run_options_test.go
View File

@ -23,10 +23,21 @@ import (
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
testRegistry := utilversion.NewComponentGlobalsRegistry()
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32))
testComponent := "test"
utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate))
testCases := []struct {
name string
testOptions *ServerRunOptions
@ -43,6 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@ -57,6 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@ -71,6 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--request-timeout can not be negative value",
},
@ -85,6 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--min-request-timeout can not be negative value",
},
@ -99,6 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@ -113,6 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@ -128,6 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@ -143,6 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@ -158,9 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
{
name: "Test when emulation version is invalid",
testOptions: &ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"},
HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentName: testComponent,
ComponentGlobalsRegistry: testRegistry,
},
expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]",
},
{
name: "Test when ServerRunOptions is valid",
testOptions: &ServerRunOptions{
@ -173,6 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
},
}

12
pkg/server/options/serving_test.go
View File

@ -43,6 +43,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
@ -276,9 +277,8 @@ func TestServerRunWithSNI(t *testing.T) {
// launch server
config := setUp(t)
v := fakeVersion()
config.Version = &v
config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String())
config.EnableIndex = true
secureOptions := (&SecureServingOptions{
@ -463,11 +463,9 @@ func certSignature(cert tls.Certificate) (string, error) {
func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
Major: "42",
Minor: "42",
GitVersion: "42.42",
}
}

111
pkg/server/storage/resource_encoding_config.go
View File

@ -21,6 +21,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/version"
)
type ResourceEncodingConfig interface {
@ -33,10 +35,15 @@ type ResourceEncodingConfig interface {
InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
}
type CompatibilityResourceEncodingConfig interface {
BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
}
type DefaultResourceEncodingConfig struct {
// resources records the overriding encoding configs for individual resources.
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
effectiveVersion version.EffectiveVersion
}
type OverridingResourceEncoding struct {
@ -47,7 +54,7 @@ type OverridingResourceEncoding struct {
var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
}
func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
}
}
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
o.effectiveVersion = effectiveVersion
}
func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group
return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
}
func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
}
// Always respect overrides
resourceOverride, resourceExists := o.resources[resource]
if resourceExists {
return resourceOverride.ExternalResourceEncoding, nil
}
return emulatedStorageVersion(
o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
example,
o.effectiveVersion,
o.scheme)
}
func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@ -82,3 +111,79 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou
}
return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
if example == nil || effectiveVersion == nil {
return binaryVersionOfResource, nil
}
// Look up example in scheme to find all objects of the same Group-Kind
// Use the highest priority version for that group-kind whose lifecycle window
// includes the current emulation version.
// If no version is found, use the binary version
// (in this case the API should be disabled anyway)
gvks, _, err := scheme.ObjectKinds(example)
if err != nil {
return schema.GroupVersion{}, err
} else if len(gvks) == 0 {
// Probably shouldn't happen if err is non-nil
return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
}
// VersionsForGroupKind returns versions in priority order
versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind})
compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
for _, gv := range versions {
if gv.Version == runtime.APIVersionInternal {
continue
}
gvk := schema.GroupVersionKind{
Group: gv.Group,
Version: gv.Version,
Kind: gvks[0].Kind,
}
exampleOfGVK, err := scheme.New(gvk)
if err != nil {
return schema.GroupVersion{}, err
}
// If it was introduced after current compatibility version, don't use it
// skip the introduced check for test when currentVersion is 0.0 to test all apis
if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
// API resource lifecycles should be relative to k8s api version
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if introducedVer.GreaterThan(compatibilityVersion) {
continue
}
}
// versions is returned in priority order, so just use first result
return gvk.GroupVersion(), nil
}
// Getting here means we're serving a version that is unknown to the
// min-compatibility-version server.
//
// This is only expected to happen when serving an alpha API type due
// to missing pre-release lifecycle information
// (which doesn't happen by default), or when emulation-version and
// min-compatibility-version are several versions apart so a beta or GA API
// was being served which didn't exist at all in min-compatibility-version.
//
// In the alpha case - we do not support compatibility versioning of
// alpha types and recommend users do not mix the two.
// In the skip-level case - The version of apiserver we are retaining
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
return binaryVersionOfResource, nil
}

19
pkg/server/storage/storage_factory.go
View File

@ -42,7 +42,7 @@ type Backend struct {
type StorageFactory interface {
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
// ResourcePrefix returns the overridden resource prefix for the GroupResource
// This allows for cohabitation of resources with different native types and provides
@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
chosenStorageResource := s.getStorageGroupResource(groupResource)
// operate on copy
@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
var err error
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
if err != nil {
return nil, err
}
} else {
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
}
}
codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
if err != nil {
return nil, err
}
codecConfig.Config = storageConfig
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)

246
pkg/server/storage/storage_factory_test.go
View File

@ -26,10 +26,12 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/apis/example"
exampleinstall "k8s.io/apiserver/pkg/apis/example/install"
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
)
var (
@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) {
f.SetEtcdLocation(example.Resource("*"), []string{"/server2"})
f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test")
config, err := f.NewConfig(example.Resource("test"))
config, err := f.NewConfig(example.Resource("test"), nil)
if err != nil {
t.Fatal(err)
}
@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
storageFactory.SetEtcdLocation(test.resource, test.servers)
var err error
config, err := storageFactory.NewConfig(test.resource)
config, err := storageFactory.NewConfig(test.resource, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
continue
}
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"})
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) {
}
}
}
var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
type fakeLifecycler[T, V any] struct {
metav1.TypeMeta
metav1.ObjectMeta
}
type removedLifecycler[T, V any] struct {
fakeLifecycler[T, V]
}
func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f }
func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f }
func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) {
if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok {
return int(introduced.Major()), int(introduced.Minor())
}
panic("no lifecycle version set")
}
func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) {
if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok {
return int(removed.Major()), int(removed.Minor())
}
panic("no lifecycle version set")
}
func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) {
f := fakeLifecycler[T, V]{}
introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced)
var res runtime.Object
if removed != "" {
removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed)
res = &removedLifecycler[T, V]{fakeLifecycler: f}
} else {
res = &f
}
var v V
var t T
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: strings.ToLower(reflect.TypeOf(v).Name()),
Kind: reflect.TypeOf(t).Name(),
},
res,
)
// Also ensure internal version is registered
// If it is registertd multiple times, it will ignore subsequent registrations
internalInstance := &fakeLifecycler[T, struct{}]{}
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: runtime.APIVersionInternal,
Kind: reflect.TypeOf(t).Name(),
},
internalInstance,
)
}
func TestStorageFactoryCompatibilityVersion(t *testing.T) {
// Creates a scheme with stub types for unit test
sch := runtime.NewScheme()
codecs := serializer.NewCodecFactory(sch)
type Internal = struct{}
type V1beta1 struct{}
type V1beta2 struct{}
type V1beta3 struct{}
type V1 struct{}
type Pod struct{}
type FlowSchema struct{}
type ValidatingAdmisisonPolicy struct{}
type CronJob struct{}
// Order dictates priority order
registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "")
registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0")
registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0")
registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0")
registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "")
registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0")
registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "")
// FlowSchema
// - v1beta1: 1.20.0 - 1.23.0
// - v1beta2: 1.23.0 - 1.26.0
// - v1beta3: 1.26.0 - 1.30.0
// - v1: 1.29.0+
// CronJob
// - v1beta1: 1.8.0 - 1.21.0
// - v1: 1.21.0+
// ValidatingAdmissionPolicy
// - v1beta1: 1.28.0 - 1.31.0
// - v1: 1.30.0+
testcases := []struct {
effectiveVersion string
example runtime.Object
expectedVersion schema.GroupVersion
}{
{
// Basic case. Beta version for long time
effectiveVersion: "1.14.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. Beta version for long time
effectiveVersion: "1.20.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. GA version for long time
effectiveVersion: "1.28.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"},
},
{
// Basic core/v1
effectiveVersion: "1.31.0",
example: &fakeLifecycler[Pod, Internal]{},
expectedVersion: schema.GroupVersion{Group: "", Version: "v1"},
},
{
// Corner case: 1.1.0 has no flowcontrol. Options are to error
// out or to use the latest version. This test assumes the latter.
effectiveVersion: "1.1.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.21.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
// v2Beta1 introduced this version, but minCompatibility should
// force v1beta1
effectiveVersion: "1.23.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.24.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.26.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.27.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// GA API introduced 1.29 but must keep storing in v1beta3 for downgrades
effectiveVersion: "1.29.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// Version after GA api is introduced
effectiveVersion: "1.30.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.30.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.31.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.29.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
}
for _, tc := range testcases {
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvk := gvks[0]
t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) {
config := NewDefaultResourceEncodingConfig(sch)
config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion))
f := NewDefaultStorageFactory(
storagebackend.Config{},
"",
codecs,
config,
NewResourceConfig(),
nil)
cfg, err := f.NewConfig(schema.GroupResource{
Group: gvk.Group,
Resource: gvk.Kind, // doesnt really matter here
}, tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion,
schema.GroupKind{
Group: gvks[0].Group,
}, schema.GroupKind{
Group: gvks[0].Group,
})
if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() {
t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner)
}
})
}
}

2
pkg/util/feature/feature_gate.go
View File

@ -25,7 +25,7 @@ var (
// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
// Tests that need to modify feature gates for the duration of their test should use:
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)
DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate()
// DefaultFeatureGate is a shared global FeatureGate.
// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.

454
pkg/util/version/registry.go Normal file
View File

@ -0,0 +1,454 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sort"
"strings"
"sync"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
DefaultKubeComponent = "kube"
klogLevel = 2
)
type VersionMapping func(from *version.Version) *version.Version
// ComponentGlobals stores the global variables for a component for easy access.
type ComponentGlobals struct {
effectiveVersion MutableEffectiveVersion
featureGate featuregate.MutableVersionedFeatureGate
// emulationVersionMapping contains the mapping from the emulation version of this component
// to the emulation version of another component.
emulationVersionMapping map[string]VersionMapping
// dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component.
// If true, the emulation version cannot be set from the flag, or version mapping from another component.
dependentEmulationVersion bool
// minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component
// to the min compatibility version of another component.
minCompatibilityVersionMapping map[string]VersionMapping
// dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component
// If true, the min compatibility version cannot be set from the flag, or version mapping from another component.
dependentMinCompatibilityVersion bool
}
type ComponentGlobalsRegistry interface {
// EffectiveVersionFor returns the EffectiveVersion registered under the component.
// Returns nil if the component is not registered.
EffectiveVersionFor(component string) EffectiveVersion
// FeatureGateFor returns the FeatureGate registered under the component.
// Returns nil if the component is not registered.
FeatureGateFor(component string) featuregate.FeatureGate
// Register registers the EffectiveVersion and FeatureGate for a component.
// returns error if the component is already registered.
Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// AddFlags adds flags of "--emulated-version" and "--feature-gates"
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
Set() error
// SetFallback calls Set() if it has never been called.
SetFallback() error
// Validate calls the Validate() function for all the global variables for all components registered.
Validate() []error
// Reset removes all stored ComponentGlobals, configurations, and version mappings.
Reset()
// SetEmulationVersionMapping sets the mapping from the emulation version of one component
// to the emulation version of another component.
// Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent,
// and cannot be set from cmd flags anymore.
// For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed.
SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]*ComponentGlobals
mutex sync.RWMutex
// list of component name to emulation version set from the flag.
emulationVersionConfig []string
// map of component name to the list of feature gates set from the flag.
featureGatesConfig map[string][]string
// set stores if the Set() function for the registry is already called.
set bool
}
func NewComponentGlobalsRegistry() *componentGlobalsRegistry {
return &componentGlobalsRegistry{
componentGlobals: make(map[string]*ComponentGlobals),
emulationVersionConfig: nil,
featureGatesConfig: nil,
}
}
func (r *componentGlobalsRegistry) Reset() {
r.mutex.RLock()
defer r.mutex.RUnlock()
r.componentGlobals = make(map[string]*ComponentGlobals)
r.emulationVersionConfig = nil
r.featureGatesConfig = nil
r.set = false
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.effectiveVersion
}
func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.featureGate
}
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if _, ok := r.componentGlobals[component]; ok {
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
c := ComponentGlobals{
effectiveVersion: effectiveVersion,
featureGate: featureGate,
emulationVersionMapping: make(map[string]VersionMapping),
minCompatibilityVersionMapping: make(map[string]VersionMapping),
}
r.componentGlobals[component] = &c
return nil
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if effectiveVersion == nil {
return fmt.Errorf("cannot register nil effectiveVersion")
}
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate)
}
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
r.mutex.Lock()
defer r.mutex.Unlock()
globals, ok := r.componentGlobals[component]
if ok {
return globals.effectiveVersion, globals.featureGate
}
utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate))
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string {
var known []string
for component, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
for _, f := range globals.featureGate.KnownFeatures() {
known = append(known, component+":"+f)
}
}
sort.Strings(known)
return known
}
func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string {
var vs []string
for component, globals := range r.componentGlobals {
binaryVer := globals.effectiveVersion.BinaryVersion()
if isEmulation {
if globals.dependentEmulationVersion {
continue
}
// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
} else {
if globals.dependentMinCompatibilityVersion {
continue
}
// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
}
}
sort.Strings(vs)
return vs
}
func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) {
if r == nil {
return
}
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
if globals.featureGate != nil {
globals.featureGate.Close()
}
}
if r.emulationVersionConfig != nil || r.featureGatesConfig != nil {
klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags")
}
r.emulationVersionConfig = []string{}
r.featureGatesConfig = make(map[string][]string)
fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+
"The versions different components emulate their capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+
"If the component is not specified, defaults to \"kube\"")
fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+
"If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+
"Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n"))
}
type componentVersion struct {
component string
ver *version.Version
}
// getFullEmulationVersionConfig expands the given version config with version registered version mapping,
// and returns the map of component to Version.
func (r *componentGlobalsRegistry) getFullEmulationVersionConfig(
versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) {
result := map[string]*version.Version{}
setQueue := []componentVersion{}
for comp, ver := range versionConfigMap {
if _, ok := r.componentGlobals[comp]; !ok {
return result, fmt.Errorf("component not registered: %s", comp)
}
klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String())
setQueue = append(setQueue, componentVersion{comp, ver})
}
for len(setQueue) > 0 {
cv := setQueue[0]
if _, visited := result[cv.component]; visited {
return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component)
}
setQueue = setQueue[1:]
result[cv.component] = cv.ver
for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping {
toVer := f(cv.ver)
if toVer == nil {
return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp)
}
klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String())
setQueue = append(setQueue, componentVersion{toComp, toVer})
}
}
return result, nil
}
func toVersionMap(versionConfig []string) (map[string]*version.Version, error) {
m := map[string]*version.Version{}
for _, compVer := range versionConfig {
// default to "kube" of component is not specified
k := "kube"
v := compVer
if strings.Contains(compVer, "=") {
arr := strings.SplitN(compVer, "=", 2)
if len(arr) != 2 {
return m, fmt.Errorf("malformed pair, expect string=string")
}
k = strings.TrimSpace(arr[0])
v = strings.TrimSpace(arr[1])
}
ver, err := version.Parse(v)
if err != nil {
return m, err
}
if ver.Patch() != 0 {
return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String())
}
if existingVer, ok := m[k]; ok {
return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String())
}
m[k] = ver
}
return m, nil
}
func (r *componentGlobalsRegistry) SetFallback() error {
r.mutex.Lock()
set := r.set
r.mutex.Unlock()
if set {
return nil
}
klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" +
" right after parsing flags to avoid using feature gates before their final values are set by the flags.")
return r.Set()
}
func (r *componentGlobalsRegistry) Set() error {
r.mutex.Lock()
defer r.mutex.Unlock()
r.set = true
emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig)
if err != nil {
return err
}
for comp := range emulationVersionConfigMap {
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
// only components without any dependencies can be set from the flag.
if r.componentGlobals[comp].dependentEmulationVersion {
return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp)
}
}
if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil {
return err
} else {
for comp, ver := range emulationVersions {
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
}
}
// Set feature gate emulation version before setting feature gate flag values.
for comp, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String())
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
for comp, fg := range r.featureGatesConfig {
if comp == "" {
if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok {
return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use")
}
comp = DefaultKubeComponent
}
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
featureGate := r.componentGlobals[comp].featureGate
if featureGate == nil {
return fmt.Errorf("component featureGate not registered: %s", comp)
}
flagVal := strings.Join(fg, ",")
klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal)
if err := featureGate.Set(flagVal); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) Validate() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
errs = append(errs, globals.effectiveVersion.Validate()...)
if globals.featureGate != nil {
errs = append(errs, globals.featureGate.Validate()...)
}
}
return errs
}
func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error {
if f == nil {
return nil
}
klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent)
r.mutex.Lock()
defer r.mutex.Unlock()
if _, ok := r.componentGlobals[fromComponent]; !ok {
return fmt.Errorf("component not registered: %s", fromComponent)
}
if _, ok := r.componentGlobals[toComponent]; !ok {
return fmt.Errorf("component not registered: %s", toComponent)
}
// check multiple dependency
if r.componentGlobals[toComponent].dependentEmulationVersion {
return fmt.Errorf("mapping of %s already exists from another component", toComponent)
}
r.componentGlobals[toComponent].dependentEmulationVersion = true
versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping
if _, ok := versionMapping[toComponent]; ok {
return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent)
}
versionMapping[toComponent] = f
klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent)
defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion()
emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion})
if err != nil {
return err
}
for comp, ver := range emulationVersions {
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
}
return nil
}

418
pkg/util/version/registry_test.go Normal file
View File

@ -0,0 +1,418 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
const (
testComponent = "test"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
if r.EffectiveVersionFor(testComponent) != nil {
t.Fatalf("expected nil EffectiveVersion initially")
}
if err := r.Register(testComponent, ver1, nil); err != nil {
t.Fatalf("expected no error to register new component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version registered")
}
// overwrite
if err := r.Register(testComponent, ver2, nil); err == nil {
t.Fatalf("expected error to register existing component when override is false")
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}
func testRegistry(t *testing.T) *componentGlobalsRegistry {
r := NewComponentGlobalsRegistry()
verKube := NewEffectiveVersion("1.31")
fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"kubeB": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
verTest := NewEffectiveVersion("2.8")
fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"testA": {
{Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
"testB": {
{Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
utilruntime.Must(r.Register(DefaultKubeComponent, verKube, fgKube))
utilruntime.Must(r.Register(testComponent, verTest, fgTest))
return r
}
func TestVersionFlagOptions(t *testing.T) {
r := testRegistry(t)
emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n")
expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionFlagOptionsWithMapping(t *testing.T) {
r := testRegistry(t)
utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent,
func(from *version.Version) *version.Version { return from.OffsetMinor(3) }))
emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n")
expectedEmuVers := "test=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionedFeatureGateFlag(t *testing.T) {
r := testRegistry(t)
known := strings.Join(r.unsafeKnownFeatures(), "\n")
expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" +
"kube:AllBeta=true|false (BETA - default=false)\n" +
"kube:commonC=true|false (BETA - default=true)\n" +
"kube:kubeB=true|false (ALPHA - default=false)\n" +
"test:AllAlpha=true|false (ALPHA - default=false)\n" +
"test:AllBeta=true|false (BETA - default=false)\n" +
"test:commonC=true|false (ALPHA - default=false)\n" +
"test:testA=true|false (BETA - default=false)"
if known != expectedKnown {
t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known)
}
}
func TestFlags(t *testing.T) {
tests := []struct {
name string
flags []string
parseError string
expectedKubeEmulationVersion string
expectedTestEmulationVersion string
expectedKubeFeatureValues map[featuregate.Feature]bool
expectedTestFeatureValues map[featuregate.Feature]bool
}{
{
name: "setting kube emulation version",
flags: []string{"--emulated-version=kube=1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "setting kube emulation version twice",
flags: []string{
"--emulated-version=kube=1.30",
"--emulated-version=kube=1.32",
},
parseError: "duplicate version flag, kube=1.30 and kube=1.32",
},
{
name: "prefix v ok",
flags: []string{"--emulated-version=kube=v1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "patch version not ok",
flags: []string{"--emulated-version=kube=1.30.2"},
parseError: "patch version not allowed, got: kube=1.30.2",
},
{
name: "setting test emulation version",
flags: []string{"--emulated-version=test=2.7"},
expectedKubeEmulationVersion: "1.31",
expectedTestEmulationVersion: "2.7",
},
{
name: "version missing component default to kube",
flags: []string{"--emulated-version=1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "version missing component default to kube with duplicate",
flags: []string{"--emulated-version=1.30", "--emulated-version=kube=1.30"},
parseError: "duplicate version flag, kube=1.30 and kube=1.30",
},
{
name: "version unregistered component",
flags: []string{"--emulated-version=test3=1.31"},
parseError: "component not registered: test3",
},
{
name: "invalid version",
flags: []string{"--emulated-version=test=1.foo"},
parseError: "illegal version string \"1.foo\"",
},
{
name: "setting test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testA=true",
},
expectedKubeEmulationVersion: "1.31",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false},
},
{
name: "setting future test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testA=true,test:testB=true",
},
parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7",
},
{
name: "setting kube feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--emulated-version=kube=1.30",
"--feature-gates=kubeB=false,test:commonC=true",
"--feature-gates=commonC=false,kubeB=true",
},
expectedKubeEmulationVersion: "1.30",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true},
},
{
name: "setting kube feature flag with different prefix",
flags: []string{
"--emulated-version=test=2.7",
"--emulated-version=kube=1.30",
"--feature-gates=kube:kubeB=false,test:commonC=true",
"--feature-gates=commonC=false,kubeB=true",
},
parseError: "set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use",
},
{
name: "setting locked kube feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=kubeA=false",
},
parseError: "cannot set feature gate kubeA to false, feature is locked to true",
},
{
name: "setting unknown test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testD=true",
},
parseError: "unrecognized feature gate: testD",
},
{
name: "setting unknown component feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test3:commonC=true",
},
parseError: "component not registered: test3",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r := testRegistry(t)
r.AddFlags(fs)
err := fs.Parse(test.flags)
if err == nil {
err = r.Set()
}
if test.parseError != "" {
if err == nil || !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() expected: nil, got: %v", i, err)
}
if len(test.expectedKubeEmulationVersion) > 0 {
assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).EmulationVersion(), test.expectedKubeEmulationVersion)
}
if len(test.expectedTestEmulationVersion) > 0 {
assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion)
}
for f, v := range test.expectedKubeFeatureValues {
if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v {
t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v)
}
}
for f, v := range test.expectedTestFeatureValues {
if r.FeatureGateFor(testComponent).Enabled(f) != v {
t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v)
}
}
})
}
}
func TestVersionMapping(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-19)
}))
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11")
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r.AddFlags(fs)
if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil {
t.Fatal(err)
return
}
if err := r.Set(); err != nil {
t.Fatal(err)
return
}
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09")
}
func TestVersionMappingWithMultipleDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
err := r.SetEmulationVersionMapping("test3", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()-1, from.Minor()+19)
})
if err == nil {
t.Errorf("expect error when setting 2nd mapping to test2")
}
}
func TestVersionMappingWithCyclicDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-19)
}))
err := r.SetEmulationVersionMapping("test3", "test1",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()-2, from.Minor()+48)
})
if err == nil {
t.Errorf("expect cyclic version mapping error")
}
}
func assertVersionEqualTo(t *testing.T, ver *version.Version, expectedVer string) {
if ver.EqualTo(version.MustParse(expectedVer)) {
return
}
t.Errorf("expected: %s, got %s", expectedVer, ver.String())
}

157
pkg/util/version/version.go Normal file
View File

@ -0,0 +1,157 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sync/atomic"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
)
type EffectiveVersion interface {
BinaryVersion() *version.Version
EmulationVersion() *version.Version
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
}
type MutableEffectiveVersion interface {
EffectiveVersion
Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version)
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
}
type effectiveVersion struct {
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion atomic.Pointer[version.Version]
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion atomic.Pointer[version.Version]
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
ver := m.emulationVersion.Load()
if ver != nil {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return ver.WithPreRelease(m.BinaryVersion().PreRelease())
}
return ver
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func majorMinor(ver *version.Version) *version.Version {
if ver == nil {
return ver
}
return version.MajorMinor(ver.Major(), ver.Minor())
}
func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) {
m.binaryVersion.Store(binaryVersion)
m.emulationVersion.Store(majorMinor(emulationVersion))
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.Store(majorMinor(emulationVersion))
}
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.binaryVersion.Load().WithPatch(0)
emulationVersion := m.emulationVersion.Load()
minCompatibilityVersion := m.minCompatibilityVersion.Load()
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(1)
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
// minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha.
maxCompVer := binaryVersion.SubtractMinor(1)
minCompVer := binaryVersion.SubtractMinor(1)
if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String()))
}
return errs
}
func newEffectiveVersion(binaryVersion *version.Version) MutableEffectiveVersion {
effective := &effectiveVersion{}
compatVersion := binaryVersion.SubtractMinor(1)
effective.Set(binaryVersion, binaryVersion, compatVersion)
return effective
}
func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion {
if binaryVer == "" {
return &effectiveVersion{}
}
binaryVersion := version.MustParse(binaryVer)
return newEffectiveVersion(binaryVersion)
}
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
verInfo := baseversion.Get()
binaryVersion := version.MustParse(verInfo.String()).WithInfo(verInfo)
if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 {
return DefaultKubeEffectiveVersion()
}
return newEffectiveVersion(binaryVersion)
}
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
return newEffectiveVersion(binaryVersion)
}

107
pkg/util/version/version_test.go Normal file
View File

@ -0,0 +1,107 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.1",
minCompatibilityVersion: "v1.31.5",
},
{
name: "emulation version one minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "emulation version two minor lower than binary not ok",
binaryVersion: "v1.33.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "emulation version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "emulation version two minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.34.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "compatibility version same as binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "compatibility version two minor lower than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "compatibility version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric(test.binaryVersion)
effective := &effectiveVersion{}
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}