kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apiserver: Add API emulation versioning. 

Co-authored-by: Siyuan Zhang <sizhang@google.com>
Co-authored-by: Joe Betz <jpbetz@google.com>
Co-authored-by: Alex Zielenski <zielenski@google.com>

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Kubernetes-commit: 403301bfdf2c7312591077827abd2e72f445a53a
This commit is contained in:
Siyuan Zhang 2024-01-19 16:07:00 -08:00 committed by Kubernetes Publisher
parent 7920da295d
commit 22612a3528
23 changed files with 1226 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/cel/environment/base.go
View File

@ -30,20 +30,27 @@ import (
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/cel/library"
utilversion "k8s.io/apiserver/pkg/util/version"
)
// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
// that guarantees compatibility with CEL features/libraries/parameters understood by
// an n-1 version
// the api server min compatibility version
//
// This default will be set to no more than n-1 the current Kubernetes major.minor version.
// This default will be set to no more than the current Kubernetes major.minor version.
//
// Note that a default version number less than n-1 indicates a wider range of version
// compatibility than strictly required for rollback. A wide range of compatibility is
// desirable because it means that CEL expressions are portable across a wider range
// of Kubernetes versions.
// Note that a default version number less than n-1 the current Kubernetes major.minor version
// indicates a wider range of version compatibility than strictly required for rollback.
// A wide range of compatibility is desirable because it means that CEL expressions are portable
// across a wider range of Kubernetes versions.
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
return version.MajorMinor(1, 30)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}
return effectiveVer.MinCompatibilityVersion()
}
var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt)

11
pkg/features/kube_features.go
View File

@ -322,6 +322,17 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
}
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}
// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

8
pkg/registry/generic/options.go
View File

@ -19,6 +19,7 @@ package generic
import (
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@ -39,12 +40,15 @@ type RESTOptions struct {
}
// Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests)
func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) {
func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) {
return opts, nil
}
type RESTOptionsGetter interface {
GetRESTOptions(resource schema.GroupResource) (RESTOptions, error)
// GetRESTOptions returns the RESTOptions for the given resource and example object.
// The example object is used to determine the storage version for the resource.
// If the example object is nil, the storage version will be determined by the resource's default storage version.
GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error)
}
// StoreOptions is set of configuration options used to complete generic registries.

2
pkg/registry/generic/registry/store.go
View File

@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
return err
}
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc())
if err != nil {
return err
}

34
pkg/server/config.go
View File

@ -42,8 +42,9 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/authenticator"
@ -70,8 +71,10 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
@ -148,7 +151,12 @@ type Config struct {
PostStartHooks map[string]PostStartHookConfigEntry
// Version will enable the /version endpoint if non-nil
Version *version.Info
Version *apimachineryversion.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// AuditBackend is where audit events are sent to.
AuditBackend audit.Backend
// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
@ -585,7 +593,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
}
}
func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
if config == nil {
return
}
@ -624,7 +632,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
}
}
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
if config == nil {
return
}
@ -676,6 +684,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
// Complete fills in any fields not set that are required to have valid data and can be derived
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
if c.FeatureGate == nil {
c.FeatureGate = utilfeature.DefaultFeatureGate
}
if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
c.ExternalAddress = c.PublicAddress.String()
}
@ -691,9 +702,12 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
}
c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
}
completeOpenAPI(c.OpenAPIConfig, c.Version)
completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
var ver *version.Version
if c.EffectiveVersion != nil {
ver = c.EffectiveVersion.EmulationVersion()
}
completeOpenAPI(c.OpenAPIConfig, ver)
completeOpenAPIV3(c.OpenAPIV3Config, ver)
if c.DiscoveryAddresses == nil {
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@ -711,7 +725,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
} else {
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
// use the storage prefix as the key if possible
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil {
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
return opts.ResourcePrefix
}
// otherwise return "" to use the default key (parent GV name)
@ -819,7 +833,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
APIServerID: c.APIServerID,
StorageVersionManager: c.StorageVersionManager,
Version: c.Version,
EffectiveVersion: c.EffectiveVersion,
Version: c.Version,
FeatureGate: c.FeatureGate,
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}

65
pkg/server/deleted_kinds.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package server
import (
"fmt"
"os"
"strconv"
"strings"
@ -25,16 +26,15 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/klog/v2"
)
// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
type resourceExpirationEvaluator struct {
currentMajor int
currentMinor int
isAlpha bool
currentVersion *apimachineryversion.Version
isAlpha bool
// This is usually set for testing for which tests need to be removed. This prevent insta-failing CI.
// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
strictRemovedHandlingInAlpha bool
@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface {
ShouldServeForVersion(majorRemoved, minorRemoved int) bool
}
func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
if currentVersion == nil {
return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
}
klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
ret := &resourceExpirationEvaluator{
strictRemovedHandlingInAlpha: false,
}
if len(currentVersion.Major) > 0 {
currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
if err != nil {
return nil, err
}
ret.currentMajor = int(currentMajor64)
}
if len(currentVersion.Minor) > 0 {
// split the "normal" + and - for semver stuff
minorString := strings.Split(currentVersion.Minor, "+")[0]
minorString = strings.Split(minorString, "-")[0]
minorString = strings.Split(minorString, ".")[0]
currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
if err != nil {
return nil, err
}
ret.currentMinor = int(currentMinor64)
}
ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
// Only keeps the major and minor versions from input version.
ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
// do nothing
@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
return false
}
introduced, ok := versionedPtr.(introducedInterface)
// skip the introduced check for test where currentVersion is 0.0
if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) {
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if e.currentVersion.LessThan(verIntroduced) {
return false
}
}
removed, ok := versionedPtr.(removedInterface)
if !ok {
return true
@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
}
func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
if e.currentMajor < majorRemoved {
removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
if removedVer.GreaterThan(e.currentVersion) {
return true
}
if e.currentMajor > majorRemoved {
return false
}
if e.currentMinor < minorRemoved {
return true
}
if e.currentMinor > minorRemoved {
if removedVer.LessThan(e.currentVersion) {
return false
}
// at this point major and minor are equal, so this API should be removed when the current release GAs.
@ -152,6 +144,11 @@ type removedInterface interface {
APILifecycleRemoved() (major, minor int)
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
}
klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
storage := versionToResource[resourceName]
storage.Destroy()
delete(versionToResource, resourceName)
}
versionedResourcesStorageMap[apiVersion] = versionToResource

173
pkg/server/deleted_kinds_test.go
View File

@ -25,57 +25,41 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/dump"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
)
func Test_newResourceExpirationEvaluator(t *testing.T) {
tests := []struct {
name string
currentVersion version.Info
currentVersion string
expected resourceExpirationEvaluator
expectedErr string
}{
{
name: "beta",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "beta",
currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "alpha",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true},
name: "alpha",
currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true},
},
{
name: "maintenance",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.1",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "maintenance",
currentVersion: "v1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "bad",
currentVersion: version.Info{
Major: "1",
Minor: "20something+",
GitVersion: "v1.20.1",
},
expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`,
name: "no v prefix",
currentVersion: "1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion)
actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion))
checkErr(t, actualErr, tt.expectedErr)
if actualErr != nil {
@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) {
}
}
func storageRemovedIn(major, minor int) removedInStorage {
return removedInStorage{major: major, minor: minor}
func storageRemovedIn(major, minor int) *removedInStorage {
return &removedInStorage{major: major, minor: minor}
}
func storageNeverRemoved() removedInStorage {
return removedInStorage{neverRemoved: true}
func storageNeverRemoved() *removedInStorage {
return &removedInStorage{neverRemoved: true}
}
type removedInStorage struct {
@ -103,23 +87,23 @@ type removedInStorage struct {
neverRemoved bool
}
func (r removedInStorage) New() runtime.Object {
func (r *removedInStorage) New() runtime.Object {
if r.neverRemoved {
return neverRemovedObj{}
return &defaultObj{}
}
return removedInObj{major: r.major, minor: r.minor}
return &removedInObj{major: r.major, minor: r.minor}
}
func (r removedInStorage) Destroy() {
func (r *removedInStorage) Destroy() {
}
type neverRemovedObj struct {
type defaultObj struct {
}
func (r neverRemovedObj) GetObjectKind() schema.ObjectKind {
func (r *defaultObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r neverRemovedObj) DeepCopyObject() runtime.Object {
func (r *defaultObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
@ -127,13 +111,45 @@ type removedInObj struct {
major, minor int
}
func (r removedInObj) GetObjectKind() schema.ObjectKind {
func (r *removedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r removedInObj) DeepCopyObject() runtime.Object {
func (r *removedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r removedInObj) APILifecycleRemoved() (major, minor int) {
func (r *removedInObj) APILifecycleRemoved() (major, minor int) {
return r.major, r.minor
}
func storageIntroducedIn(major, minor int) *introducedInStorage {
return &introducedInStorage{major: major, minor: minor}
}
type introducedInStorage struct {
major, minor int
}
func (r *introducedInStorage) New() runtime.Object {
if r.major == 0 && r.minor == 0 {
return &defaultObj{}
}
return &IntroducedInObj{major: r.major, minor: r.minor}
}
func (r *introducedInStorage) Destroy() {
}
type IntroducedInObj struct {
major, minor int
}
func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r *IntroducedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) {
return r.major, r.minor
}
@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 20),
expected: false,
@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-deferred",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
isAlpha: true,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
},
restStorage: storageRemovedIn(1, 20),
expected: true,
@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha-but-strict",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
strictRemovedHandlingInAlpha: true,
},
@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-deferral-does-not-help",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 21,
currentVersion: apimachineryversion.MajorMinor(1, 21),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 2,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(2, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 21),
expected: true,
@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "never-removed",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageNeverRemoved(),
expected: true,
},
{
name: "introduced-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 20),
expected: true,
},
{
name: "introduced-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 19),
expected: true,
},
{
name: "introduced-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 21),
expected: false,
},
{
name: "missing-introduced",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(0, 0),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-one-of-two",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-nested-not-expired",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-all-of-version",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {

7
pkg/server/genericapiserver.go
View File

@ -53,7 +53,9 @@ import (
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
openapicommon "k8s.io/kube-openapi/pkg/common"
@ -238,6 +240,11 @@ type GenericAPIServer struct {
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
lifecycleSignals lifecycleSignals

4
pkg/server/genericapiserver_test.go
View File

@ -48,6 +48,7 @@ import (
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
"k8s.io/apiserver/pkg/registry/rest"
genericfilters "k8s.io/apiserver/pkg/server/filters"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
@ -460,6 +461,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) {
kubeVersion := fakeVersion()
config.Version = &kubeVersion
config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String())
s, err := config.Complete(nil).New("test", NewEmptyDelegate())
if err != nil {
@ -586,7 +588,7 @@ func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitVersion: "42.42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
BuildDate: time.Now().String(),

42
pkg/server/options/api_enablement_test.go
View File

@ -32,46 +32,42 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
func TestAPIEnablementOptionsValidate(t *testing.T) {
testCases := []struct {
name string
testOptions *APIEnablementOptions
expectErr string
name string
testOptions *APIEnablementOptions
runtimeConfig cliflag.ConfigurationMap
expectErr string
}{
{
name: "test when options is nil",
},
{
name: "test when invalid key with only api/all=false",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
},
expectErr: "invalid key with only api/all=false",
name: "test when invalid key with only api/all=false",
runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
expectErr: "invalid key with only api/all=false",
},
{
name: "test when ConfigurationMap key is invalid",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
},
expectErr: "runtime-config invalid key",
name: "test when ConfigurationMap key is invalid",
runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
expectErr: "runtime-config invalid key",
},
{
name: "test when unknown api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
},
expectErr: "unknown api groups",
name: "test when unknown api groups",
runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
expectErr: "unknown api groups",
},
{
name: "test when valid api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
name: "test when valid api groups",
runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
}
testGroupRegistry := fakeGroupRegistry{}
for _, testcase := range testCases {
t.Run(testcase.name, func(t *testing.T) {
errs := testcase.testOptions.Validate(testGroupRegistry)
testOptions := &APIEnablementOptions{
RuntimeConfig: testcase.runtimeConfig,
}
errs := testOptions.Validate(testGroupRegistry)
if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) {
t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr)
}

10
pkg/server/options/etcd.go
View File

@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
StorageFactory serverstorage.StorageFactory
}
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource)
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
}
@ -469,7 +469,7 @@ type SimpleStorageFactory struct {
StorageConfig storagebackend.Config
}
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return s.StorageConfig.ForResource(resource), nil
}
@ -493,8 +493,8 @@ type transformerStorageFactory struct {
resourceTransformers storagevalue.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource)
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource, example)
if err != nil {
return nil, err
}

2
pkg/server/options/etcd_test.go
View File

@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) {
if err := etcdOptions.ApplyTo(serverConfig); err != nil {
t.Fatalf("Failed to apply etcd options error: %v", err)
}
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

34
pkg/server/options/server_run_options.go
View File

@ -26,7 +26,8 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"github.com/spf13/pflag"
)
@ -89,9 +90,13 @@ type ServerRunOptions struct {
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
// FeatureGate are the featuregate to install on the CLI
FeatureGate featuregate.FeatureGate
EffectiveVersion utilversion.EffectiveVersion
}
func NewServerRunOptions() *ServerRunOptions {
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
@ -104,6 +109,8 @@ func NewServerRunOptions() *ServerRunOptions {
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
}
}
@ -124,6 +131,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
c.EffectiveVersion = s.EffectiveVersion
c.FeatureGate = s.FeatureGate
return nil
}
@ -196,6 +205,14 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
if s.FeatureGate != nil {
if errs := s.FeatureGate.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
}
if errs := s.EffectiveVersion.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
return errors
}
@ -336,6 +353,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
// Complete fills missing fields with defaults.
func (s *ServerRunOptions) Complete() error {
if s.FeatureGate == nil {
return fmt.Errorf("nil FeatureGate in ServerRunOptions")
}
if s.EffectiveVersion == nil {
return fmt.Errorf("nil EffectiveVersion in ServerRunOptions")
}
return nil
}

36
pkg/server/options/server_run_options_test.go
View File

@ -23,10 +23,14 @@ import (
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
testCases := []struct {
name string
testOptions *ServerRunOptions
@ -43,6 +47,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@ -57,6 +63,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@ -71,6 +79,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--request-timeout can not be negative value",
},
@ -85,6 +95,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--min-request-timeout can not be negative value",
},
@ -99,6 +111,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@ -113,6 +127,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@ -128,6 +144,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@ -143,6 +161,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@ -158,6 +178,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
@ -173,6 +195,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
},
}
@ -192,6 +216,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
}
func TestValidateCorsAllowedOriginList(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
regexp [][]string
errShouldContain string
@ -239,7 +265,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
for _, test := range tests {
for _, regexp := range test.regexp {
t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) {
options := NewServerRunOptions()
options := NewServerRunOptions(featureGate, effectiveVersion)
if errs := options.Validate(); len(errs) != 0 {
t.Fatalf("wrong test setup: %#v", errs)
}
@ -263,6 +289,8 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
}
func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
name string
optionsFn func() *ServerRunOptions
@ -271,13 +299,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
{
name: "default should be valid",
optionsFn: func() *ServerRunOptions {
return NewServerRunOptions()
return NewServerRunOptions(featureGate, effectiveVersion)
},
},
{
name: "negative not allowed",
optionsFn: func() *ServerRunOptions {
o := NewServerRunOptions()
o := NewServerRunOptions(featureGate, effectiveVersion)
o.ShutdownWatchTerminationGracePeriod = -time.Second
return o
},
@ -304,7 +332,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
}
t.Run("default should be zero", func(t *testing.T) {
options := NewServerRunOptions()
options := NewServerRunOptions(featureGate, effectiveVersion)
if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) {
t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod)
}

11
pkg/server/options/serving_test.go
View File

@ -43,6 +43,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
@ -276,9 +277,9 @@ func TestServerRunWithSNI(t *testing.T) {
// launch server
config := setUp(t)
v := fakeVersion()
config.Version = &v
config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String())
config.EnableIndex = true
secureOptions := (&SecureServingOptions{
@ -463,11 +464,9 @@ func certSignature(cert tls.Certificate) (string, error) {
func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
Major: "42",
Minor: "42",
GitVersion: "42.42",
}
}

110
pkg/server/storage/resource_encoding_config.go
View File

@ -21,6 +21,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/version"
)
type ResourceEncodingConfig interface {
@ -33,10 +35,15 @@ type ResourceEncodingConfig interface {
InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
}
type CompatibilityResourceEncodingConfig interface {
BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
}
type DefaultResourceEncodingConfig struct {
// resources records the overriding encoding configs for individual resources.
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
effectiveVersion version.EffectiveVersion
}
type OverridingResourceEncoding struct {
@ -47,7 +54,7 @@ type OverridingResourceEncoding struct {
var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
}
func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
}
}
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
o.effectiveVersion = effectiveVersion
}
func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group
return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
}
func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
}
// Always respect overrides
resourceOverride, resourceExists := o.resources[resource]
if resourceExists {
return resourceOverride.ExternalResourceEncoding, nil
}
return emulatedStorageVersion(
o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
example,
o.effectiveVersion,
o.scheme)
}
func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@ -82,3 +111,78 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou
}
return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
if example == nil || effectiveVersion == nil {
return binaryVersionOfResource, nil
}
// Look up example in scheme to find all objects of the same Group-Kind
// Use the highest priority version for that group-kind whose lifecycle window
// includes the current emulation version.
// If no version is found, use the binary version
// (in this case the API should be disabled anyway)
gvks, _, err := scheme.ObjectKinds(example)
if err != nil {
return schema.GroupVersion{}, err
} else if len(gvks) == 0 {
// Probably shouldn't happen if err is non-nil
return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
}
// VersionsForGroupKind returns versions in priority order
versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind})
compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
for _, gv := range versions {
if gv.Version == runtime.APIVersionInternal {
continue
}
gvk := schema.GroupVersionKind{
Group: gv.Group,
Version: gv.Version,
Kind: gvks[0].Kind,
}
exampleOfGVK, err := scheme.New(gvk)
if err != nil {
return schema.GroupVersion{}, err
}
// If it was introduced after current compatibility version, don't use it
if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
// API resource lifecycles should be relative to k8s api version
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if introducedVer.GreaterThan(compatibilityVersion) {
continue
}
}
// versions is returned in priority order, so just use first result
return gvk.GroupVersion(), nil
}
// Getting here means we're serving a version that is unknown to the
// min-compatibility-version server.
//
// This is only expected to happen when serving an alpha API type due
// to missing pre-release lifecycle information
// (which doesn't happen by default), or when emulation-version and
// min-compatibility-version are several versions apart so a beta or GA API
// was being served which didn't exist at all in min-compatibility-version.
//
// In the alpha case - we do not support compatibility versioning of
// alpha types and recommend users do not mix the two.
// In the skip-level case - The version of apiserver we are retaining
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
return binaryVersionOfResource, nil
}

19
pkg/server/storage/storage_factory.go
View File

@ -42,7 +42,7 @@ type Backend struct {
type StorageFactory interface {
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
// ResourcePrefix returns the overridden resource prefix for the GroupResource
// This allows for cohabitation of resources with different native types and provides
@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
chosenStorageResource := s.getStorageGroupResource(groupResource)
// operate on copy
@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
var err error
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
if err != nil {
return nil, err
}
} else {
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
}
}
codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
if err != nil {
return nil, err
}
codecConfig.Config = storageConfig
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)

246
pkg/server/storage/storage_factory_test.go
View File

@ -26,10 +26,12 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/apis/example"
exampleinstall "k8s.io/apiserver/pkg/apis/example/install"
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
)
var (
@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) {
f.SetEtcdLocation(example.Resource("*"), []string{"/server2"})
f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test")
config, err := f.NewConfig(example.Resource("test"))
config, err := f.NewConfig(example.Resource("test"), nil)
if err != nil {
t.Fatal(err)
}
@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
storageFactory.SetEtcdLocation(test.resource, test.servers)
var err error
config, err := storageFactory.NewConfig(test.resource)
config, err := storageFactory.NewConfig(test.resource, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
continue
}
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"})
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) {
}
}
}
var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
type fakeLifecycler[T, V any] struct {
metav1.TypeMeta
metav1.ObjectMeta
}
type removedLifecycler[T, V any] struct {
fakeLifecycler[T, V]
}
func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f }
func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f }
func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) {
if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok {
return int(introduced.Major()), int(introduced.Minor())
}
panic("no lifecycle version set")
}
func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) {
if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok {
return int(removed.Major()), int(removed.Minor())
}
panic("no lifecycle version set")
}
func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) {
f := fakeLifecycler[T, V]{}
introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced)
var res runtime.Object
if removed != "" {
removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed)
res = &removedLifecycler[T, V]{fakeLifecycler: f}
} else {
res = &f
}
var v V
var t T
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: strings.ToLower(reflect.TypeOf(v).Name()),
Kind: reflect.TypeOf(t).Name(),
},
res,
)
// Also ensure internal version is registered
// If it is registertd multiple times, it will ignore subsequent registrations
internalInstance := &fakeLifecycler[T, struct{}]{}
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: runtime.APIVersionInternal,
Kind: reflect.TypeOf(t).Name(),
},
internalInstance,
)
}
func TestStorageFactoryCompatibilityVersion(t *testing.T) {
// Creates a scheme with stub types for unit test
sch := runtime.NewScheme()
codecs := serializer.NewCodecFactory(sch)
type Internal = struct{}
type V1beta1 struct{}
type V1beta2 struct{}
type V1beta3 struct{}
type V1 struct{}
type Pod struct{}
type FlowSchema struct{}
type ValidatingAdmisisonPolicy struct{}
type CronJob struct{}
// Order dictates priority order
registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "")
registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0")
registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0")
registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0")
registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "")
registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0")
registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "")
// FlowSchema
// - v1beta1: 1.20.0 - 1.23.0
// - v1beta2: 1.23.0 - 1.26.0
// - v1beta3: 1.26.0 - 1.30.0
// - v1: 1.29.0+
// CronJob
// - v1beta1: 1.8.0 - 1.21.0
// - v1: 1.21.0+
// ValidatingAdmissionPolicy
// - v1beta1: 1.28.0 - 1.31.0
// - v1: 1.30.0+
testcases := []struct {
effectiveVersion string
example runtime.Object
expectedVersion schema.GroupVersion
}{
{
// Basic case. Beta version for long time
effectiveVersion: "1.14.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. Beta version for long time
effectiveVersion: "1.20.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. GA version for long time
effectiveVersion: "1.28.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"},
},
{
// Basic core/v1
effectiveVersion: "1.31.0",
example: &fakeLifecycler[Pod, Internal]{},
expectedVersion: schema.GroupVersion{Group: "", Version: "v1"},
},
{
// Corner case: 1.1.0 has no flowcontrol. Options are to error
// out or to use the latest version. This test assumes the latter.
effectiveVersion: "1.1.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.21.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
// v2Beta1 introduced this version, but minCompatibility should
// force v1beta1
effectiveVersion: "1.23.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.24.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.26.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.27.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// GA API introduced 1.29 but must keep storing in v1beta3 for downgrades
effectiveVersion: "1.29.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// Version after GA api is introduced
effectiveVersion: "1.30.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.30.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.31.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.29.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
}
for _, tc := range testcases {
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvk := gvks[0]
t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) {
config := NewDefaultResourceEncodingConfig(sch)
config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion))
f := NewDefaultStorageFactory(
storagebackend.Config{},
"",
codecs,
config,
NewResourceConfig(),
nil)
cfg, err := f.NewConfig(schema.GroupResource{
Group: gvk.Group,
Resource: gvk.Kind, // doesnt really matter here
}, tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion,
schema.GroupKind{
Group: gvks[0].Group,
}, schema.GroupKind{
Group: gvks[0].Group,
})
if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() {
t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner)
}
})
}
}

2
pkg/util/feature/feature_gate.go
View File

@ -25,7 +25,7 @@ var (
// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
// Tests that need to modify feature gates for the duration of their test should use:
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)
DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate()
// DefaultFeatureGate is a shared global FeatureGate.
// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.

142
pkg/util/version/registry.go Normal file
View File

@ -0,0 +1,142 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sync"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/component-base/featuregate"
)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
ComponentGenericAPIServer = "k8s.io/apiserver"
)
// ComponentGlobals stores the global variables for a component for easy access.
type ComponentGlobals struct {
effectiveVersion MutableEffectiveVersion
featureGate featuregate.MutableVersionedFeatureGate
}
type ComponentGlobalsRegistry interface {
// EffectiveVersionFor returns the EffectiveVersion registered under the component.
// Returns nil if the component is not registered.
EffectiveVersionFor(component string) EffectiveVersion
// FeatureGateFor returns the FeatureGate registered under the component.
// Returns nil if the component is not registered.
FeatureGateFor(component string) featuregate.FeatureGate
// Register registers the EffectiveVersion and FeatureGate for a component.
// Overrides existing ComponentGlobals if it is already in the registry if override is true,
// otherwise returns error if the component is already registered.
Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// SetAllComponents sets the emulation version for other global variables for all components registered.
SetAllComponents() error
// SetAllComponents calls the Validate() function for all the global variables for all components registered.
ValidateAllComponents() []error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]ComponentGlobals
mutex sync.RWMutex
}
func NewComponentGlobalsRegistry() ComponentGlobalsRegistry {
return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}}
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.effectiveVersion
}
func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.featureGate
}
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
if _, ok := r.componentGlobals[component]; ok && !override {
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
featureGate.DeferErrorsToValidation(true)
}
c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate}
r.componentGlobals[component] = c
return nil
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate, override)
}
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
r.mutex.Lock()
defer r.mutex.Unlock()
globals, ok := r.componentGlobals[component]
if ok {
return globals.effectiveVersion, globals.featureGate
}
utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false))
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) SetAllComponents() error {
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) ValidateAllComponents() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
errs = append(errs, globals.effectiveVersion.Validate()...)
if globals.featureGate != nil {
errs = append(errs, globals.featureGate.Validate()...)
}
}
return errs
}

48
pkg/util/version/registry_test.go Normal file
View File

@ -0,0 +1,48 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"testing"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
testComponent := "test"
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
if r.EffectiveVersionFor(testComponent) != nil {
t.Fatalf("expected nil EffectiveVersion initially")
}
if err := r.Register(testComponent, ver1, nil, false); err != nil {
t.Fatalf("expected no error to register new component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version registered")
}
// overwrite
if err := r.Register(testComponent, ver2, nil, false); err == nil {
t.Fatalf("expected error to register existing component when override is false")
}
if err := r.Register(testComponent, ver2, nil, true); err != nil {
t.Fatalf("expected no error to overriding existing component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}

195
pkg/util/version/version.go Normal file
View File

@ -0,0 +1,195 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"sync/atomic"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
)
type EffectiveVersion interface {
BinaryVersion() *version.Version
EmulationVersion() *version.Version
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
}
type MutableEffectiveVersion interface {
EffectiveVersion
Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version)
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
// AddFlags adds the "{prefix}-emulated-version" to the flagset.
AddFlags(fs *pflag.FlagSet, prefix string)
}
type VersionVar struct {
val atomic.Pointer[version.Version]
}
// Set sets the flag value
func (v *VersionVar) Set(s string) error {
components := strings.Split(s, ".")
if len(components) != 2 {
return fmt.Errorf("version %s is not in the format of major.minor", s)
}
ver, err := version.ParseGeneric(s)
if err != nil {
return err
}
v.val.Store(ver)
return nil
}
// String returns the flag value
func (v *VersionVar) String() string {
ver := v.val.Load()
return ver.String()
}
// Type gets the flag type
func (v *VersionVar) Type() string {
return "version"
}
type effectiveVersion struct {
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion VersionVar
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion VersionVar
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease())
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.val.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) {
m.binaryVersion.Store(binaryVersion)
m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor()))
m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor()))
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor()))
}
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor()))
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.binaryVersion.Load().WithPatch(0)
emulationVersion := m.emulationVersion.val.Load()
minCompatibilityVersion := m.minCompatibilityVersion.val.Load()
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(1)
// TODO: remove in 1.32
// emulationVersion is introduced in 1.31, so it cannot be lower than that.
// binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31.
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) {
minEmuVer = version.MajorMinor(1, 31)
}
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
// minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha.
maxCompVer := binaryVersion.SubtractMinor(1)
minCompVer := binaryVersion.SubtractMinor(1)
if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String()))
}
return errs
}
// AddFlags adds the "{prefix}-emulated-version" to the flagset.
func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) {
if m == nil {
return
}
if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}
fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+
"The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+
"This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+
"Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+
"Format could only be major.minor")
}
func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion {
effective := &effectiveVersion{}
binaryVersion := version.MustParse(binaryVer)
compatVersion := binaryVersion.SubtractMinor(1)
effective.Set(binaryVersion, binaryVersion, compatVersion)
return effective
}
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
verInfo := baseversion.Get()
ver := NewEffectiveVersion(verInfo.String())
if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 {
ver = DefaultKubeEffectiveVersion()
}
return ver
}
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
// Should update for each minor release!
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
return NewEffectiveVersion("1.31")
}

180
pkg/util/version/version_test.go Normal file
View File

@ -0,0 +1,180 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.1",
minCompatibilityVersion: "v1.31.5",
},
{
name: "emulation version one minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "binary version 1.31, emulation version lower than 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "binary version 1.31, emulation version 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.30.0",
},
{
name: "binary version lower than 1.31",
binaryVersion: "v1.30.2",
emulationVersion: "v1.29.0",
minCompatibilityVersion: "v1.29.0",
},
{
name: "emulation version two minor lower than binary not ok",
binaryVersion: "v1.33.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "emulation version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "emulation version two minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.34.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "compatibility version same as binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "compatibility version two minor lower than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "compatibility version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric(test.binaryVersion)
effective := &effectiveVersion{}
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}
func TestEffectiveVersionsFlag(t *testing.T) {
tests := []struct {
name string
emulationVerson string
expectedEmulationVersion *version.Version
parseError string
}{
{
name: "major.minor ok",
emulationVerson: "1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "v prefix ok",
emulationVerson: "v1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "semantic version not ok",
emulationVerson: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
},
{
name: "invalid version",
emulationVerson: "1.foo",
parseError: "illegal version string",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
effective := NewEffectiveVersion("1.30")
effective.AddFlags(fs, "test")
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)})
if test.parseError != "" {
if !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() Expected nil, Got %v", i, err)
}
if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) {
t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String())
}
})
}
}