kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: fix version order in emulation forward compatibility. 

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Kubernetes-commit: b1a9cc347311012bade7230b55ab95229a1e22c9
This commit is contained in:
Siyuan Zhang 2025-04-07 18:38:43 -07:00 committed by Kubernetes Publisher
parent e87c9dbf8f
commit 9ca332f5c7
7 changed files with 287 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/server/config.go
View File

@ -156,8 +156,9 @@ type Config struct {
EffectiveVersion basecompatibility.EffectiveVersion
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

6
pkg/server/deleted_kinds.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"os"
"regexp"
"sort"
"strconv"
"strings"
@ -28,6 +29,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/registry/rest"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/klog/v2"
@ -71,6 +73,7 @@ type ResourceExpirationEvaluatorOptions struct {
Prerelease string
// EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
// when resources of the same group and resource name but with lower priority are served.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
// when the resource is explicitly enabled in runtime-config.
@ -222,6 +225,9 @@ func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, v
func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
versionsToRemove := sets.NewString()
prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
enabledResources := sets.NewString()
// iterate from the end to the front, so that we remove the lower priority versions first.

261
pkg/server/deleted_kinds_test.go
View File

@ -801,13 +801,272 @@ func Test_removeUnIntroducedKinds(t *testing.T) {
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 19),
runtimeConfigEmulationForwardCompatible: true,
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2alpha1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 19),
runtimeConfigEmulationForwardCompatible: true,
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2alpha1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
},
},
},
{
name: "emulation-forward-compatible-beta1-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 21),
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "emulation-forward-compatible-beta1-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 21),
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
emulationForwardCompatible: true,
runtimeConfigEmulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
emulationForwardCompatible: true,
runtimeConfigEmulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resourceConfig := serverstorage.NewResourceConfig()
convertor := &dummyConvertor{prioritizedVersions: []schema.GroupVersion{
{Group: groupName, Version: "v2"}, {Group: groupName, Version: "v1"},
{Group: groupName, Version: "v2beta2"}, {Group: groupName, Version: "v2beta1"},
{Group: groupName, Version: "v2beta1"},
{Group: groupName, Version: "v2beta2"},
{Group: groupName, Version: "v2alpha1"}}}
resourceConfig.EnableVersions(convertor.PrioritizedVersionsForGroup(groupName)...)
resourceConfig, err := resourceconfig.MergeAPIResourceConfigs(resourceConfig, tt.runtimeConfig, convertor)

3
pkg/server/genericapiserver.go
View File

@ -247,8 +247,9 @@ type GenericAPIServer struct {
EffectiveVersion basecompatibility.EffectiveVersion
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

6
pkg/server/options/server_run_options.go
View File

@ -100,8 +100,9 @@ type ServerRunOptions struct {
ComponentName string
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
@ -399,10 +400,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
s.ComponentGlobalsRegistry.AddFlags(fs)
fs.BoolVar(&s.EmulationForwardCompatible, "emulation-forward-compatible", s.EmulationForwardCompatible, ""+
"If true all APIs that have higher priority than the APIs enabled at the emulation version of the same group resource will be installed. "+
"If true, for any beta+ APIs enabled by default or by --runtime-config at the emulation version, their future versions with higher priority/stability will be auto enabled even if they introduced after the emulation version. "+
"Can only be set to true if the emulation version is lower than the binary version.")
fs.BoolVar(&s.RuntimeConfigEmulationForwardCompatible, "runtime-config-emulation-forward-compatible", s.RuntimeConfigEmulationForwardCompatible, ""+
"If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. "+
"If false, server would fail to start if any APIs identified by group/version that are enabled in the --runtime-config flag are introduced after the emulation version. "+
"Can only be set to true if the emulation version is lower than the binary version.")
}

14
pkg/server/resourceconfig/helpers.go
View File

@ -19,11 +19,13 @@ package resourceconfig
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/version"
serverstore "k8s.io/apiserver/pkg/server/storage"
cliflag "k8s.io/component-base/cli/flag"
)
@ -258,7 +260,11 @@ func EmulationForwardCompatibleResourceConfig(
continue
}
// if a gv is enabled, all the versions with higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
for _, pgv := range registry.PrioritizedVersionsForGroup(gv.Group) {
prioritizedVersions := registry.PrioritizedVersionsForGroup(gv.Group)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
for _, pgv := range prioritizedVersions {
if pgv.Version == gv.Version {
break
}
@ -275,7 +281,11 @@ func EmulationForwardCompatibleResourceConfig(
continue
}
// if a gvr is enabled, all the versions with the same resource name and higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
for _, pgv := range registry.PrioritizedVersionsForGroup(gvr.Group) {
prioritizedVersions := registry.PrioritizedVersionsForGroup(gvr.Group)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
for _, pgv := range prioritizedVersions {
if pgv.Version == gvr.Version {
break
}

2
pkg/server/resourceconfig/helpers_test.go
View File

@ -802,5 +802,5 @@ func addTestGVs(t *testing.T, s *runtime.Scheme) {
s.AddKnownTypes(v2, &runtimetesting.TestType1{}, &runtimetesting.TestType2{})
require.NoError(t, runtimetesting.RegisterConversions(s))
require.NoError(t, s.SetVersionPriority(v2, v1, v2beta2, v2beta1))
require.NoError(t, s.SetVersionPriority(v2, v1, v2beta1, v2beta2))
}