kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add composition flags for emulation version and feature gate. 

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Kubernetes-commit: 701e5fc3744c2b21ba5b1ca3399b71c9becf06d0
This commit is contained in:
Siyuan Zhang 2024-05-30 12:08:52 -07:00 committed by Kubernetes Publisher
parent 22612a3528
commit c80af88d21
4 changed files with 379 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/cel/environment/base.go
View File

@ -46,7 +46,7 @@ import (
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}

153
pkg/util/version/registry.go
View File

@ -18,16 +18,47 @@ package version
import (
"fmt"
"sort"
"strings"
"sync"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
ComponentGenericAPIServer = "k8s.io/apiserver"
DefaultKubeComponent = "kube"
)
// ComponentGlobals stores the global variables for a component for easy access.
@ -50,19 +81,27 @@ type ComponentGlobalsRegistry interface {
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// SetAllComponents sets the emulation version for other global variables for all components registered.
SetAllComponents() error
// AddFlags adds flags of "--emulated-version" and "--feature-gates"
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
Set() error
// SetAllComponents calls the Validate() function for all the global variables for all components registered.
ValidateAllComponents() []error
Validate() []error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]ComponentGlobals
mutex sync.RWMutex
// map of component name to emulation version set from the flag.
emulationVersionConfig cliflag.ConfigurationMap
// map of component name to the list of feature gates set from the flag.
featureGatesConfig map[string][]string
}
func NewComponentGlobalsRegistry() ComponentGlobalsRegistry {
return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}}
return &componentGlobalsRegistry{
componentGlobals: make(map[string]ComponentGlobals),
}
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
@ -90,7 +129,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
featureGate.DeferErrorsToValidation(true)
if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate}
r.componentGlobals[component] = c
@ -98,6 +139,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
if effectiveVersion == nil {
return fmt.Errorf("cannot register nil effectiveVersion")
}
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate, override)
@ -114,21 +158,112 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string,
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) SetAllComponents() error {
func (r *componentGlobalsRegistry) knownFeatures() []string {
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
var known []string
for component, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
for _, f := range globals.featureGate.KnownFeatures() {
known = append(known, component+":"+f)
}
}
sort.Strings(known)
return known
}
func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string {
r.mutex.Lock()
defer r.mutex.Unlock()
var vs []string
for component, globals := range r.componentGlobals {
binaryVer := globals.effectiveVersion.BinaryVersion()
if isEmulation {
// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
} else {
// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
}
}
sort.Strings(vs)
return vs
}
func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) {
if r == nil {
return
}
r.mutex.Lock()
for _, globals := range r.componentGlobals {
if globals.featureGate != nil {
globals.featureGate.Close()
}
}
r.emulationVersionConfig = make(cliflag.ConfigurationMap)
r.featureGatesConfig = make(map[string][]string)
r.mutex.Unlock()
fs.Var(&r.emulationVersionConfig, "emulated-version", ""+
"The versions different components emulate their capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n"))
fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+
"If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+
"Options are:\n"+strings.Join(r.knownFeatures(), "\n"))
}
func (r *componentGlobalsRegistry) Set() error {
r.mutex.Lock()
defer r.mutex.Unlock()
for comp, emuVer := range r.emulationVersionConfig {
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer)
v, err := version.Parse(emuVer)
if err != nil {
return err
}
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v)
}
// Set feature gate emulation version before setting feature gate flag values.
for comp, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String())
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
for comp, fg := range r.featureGatesConfig {
if comp == "" {
comp = DefaultKubeComponent
}
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
featureGate := r.componentGlobals[comp].featureGate
if featureGate == nil {
return fmt.Errorf("component featureGate not registered: %s", comp)
}
flagVal := strings.Join(fg, ",")
klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal)
if err := featureGate.Set(flagVal); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) ValidateAllComponents() []error {
func (r *componentGlobalsRegistry) Validate() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()

225
pkg/util/version/registry_test.go
View File

@ -17,12 +17,22 @@ limitations under the License.
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
)
const (
testComponent = "test"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
testComponent := "test"
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
@ -46,3 +56,216 @@ func TestEffectiveVersionRegistry(t *testing.T) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}
func testRegistry(t *testing.T) *componentGlobalsRegistry {
r := componentGlobalsRegistry{
componentGlobals: map[string]ComponentGlobals{},
emulationVersionConfig: make(cliflag.ConfigurationMap),
featureGatesConfig: make(map[string][]string),
}
verKube := NewEffectiveVersion("1.31")
fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"kubeB": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
verTest := NewEffectiveVersion("2.8")
fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"testA": {
{Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
"testB": {
{Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
_ = r.Register(DefaultKubeComponent, verKube, fgKube, true)
_ = r.Register(testComponent, verTest, fgTest, true)
return &r
}
func TestVersionFlagOptions(t *testing.T) {
r := testRegistry(t)
emuVers := strings.Join(r.versionFlagOptions(true), "\n")
expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.versionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionedFeatureGateFlag(t *testing.T) {
r := testRegistry(t)
known := strings.Join(r.knownFeatures(), "\n")
expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" +
"kube:AllBeta=true|false (BETA - default=false)\n" +
"kube:commonC=true|false (BETA - default=true)\n" +
"kube:kubeB=true|false (ALPHA - default=false)\n" +
"test:AllAlpha=true|false (ALPHA - default=false)\n" +
"test:AllBeta=true|false (BETA - default=false)\n" +
"test:commonC=true|false (ALPHA - default=false)\n" +
"test:testA=true|false (BETA - default=false)"
if known != expectedKnown {
t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known)
}
}
func TestFlags(t *testing.T) {
tests := []struct {
name string
emulationVersionFlag string
featureGatesFlag string
parseError string
expectedKubeEmulationVersion *version.Version
expectedTestEmulationVersion *version.Version
expectedKubeFeatureValues map[featuregate.Feature]bool
expectedTestFeatureValues map[featuregate.Feature]bool
}{
{
name: "setting kube emulation version",
emulationVersionFlag: "kube=1.30",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "setting kube emulation version, prefix v ok",
emulationVersionFlag: "kube=v1.30",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "setting test emulation version",
emulationVersionFlag: "test=2.7",
expectedKubeEmulationVersion: version.MajorMinor(1, 31),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
},
{
name: "version missing component",
emulationVersionFlag: "1.31",
parseError: "component not registered: 1.31",
},
{
name: "version unregistered component",
emulationVersionFlag: "test3=1.31",
parseError: "component not registered: test3",
},
{
name: "invalid version",
emulationVersionFlag: "test=1.foo",
parseError: "illegal version string \"1.foo\"",
},
{
name: "setting test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testA=true",
expectedKubeEmulationVersion: version.MajorMinor(1, 31),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false},
},
{
name: "setting future test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testA=true,test:testB=true",
parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7",
},
{
name: "setting kube feature flag",
emulationVersionFlag: "test=2.7,kube=1.30",
featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true},
},
{
name: "setting locked kube feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "kubeA=false",
parseError: "cannot set feature gate kubeA to false, feature is locked to true",
},
{
name: "setting unknown test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testD=true",
parseError: "unrecognized feature gate: testD",
},
{
name: "setting unknown component feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test3:commonC=true",
parseError: "component not registered: test3",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r := testRegistry(t)
r.AddFlags(fs)
err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag),
fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)})
if err == nil {
err = r.Set()
}
if test.parseError != "" {
if err == nil || !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() expected: nil, got: %v", i, err)
}
if test.expectedKubeEmulationVersion != nil {
v := r.EffectiveVersionFor("kube").EmulationVersion()
if !v.EqualTo(test.expectedKubeEmulationVersion) {
t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String())
return
}
}
if test.expectedTestEmulationVersion != nil {
v := r.EffectiveVersionFor("test").EmulationVersion()
if !v.EqualTo(test.expectedTestEmulationVersion) {
t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String())
return
}
}
for f, v := range test.expectedKubeFeatureValues {
if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v {
t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v)
}
}
for f, v := range test.expectedTestFeatureValues {
if r.FeatureGateFor(testComponent).Enabled(f) != v {
t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v)
}
}
})
}
}

20
pkg/util/version/version_test.go
View File

@ -131,29 +131,29 @@ func TestValidate(t *testing.T) {
func TestEffectiveVersionsFlag(t *testing.T) {
tests := []struct {
name string
emulationVerson string
emulationVersion string
expectedEmulationVersion *version.Version
parseError string
}{
{
name: "major.minor ok",
emulationVerson: "1.30",
emulationVersion: "1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "v prefix ok",
emulationVerson: "v1.30",
emulationVersion: "v1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "semantic version not ok",
emulationVerson: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
name: "semantic version not ok",
emulationVersion: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
},
{
name: "invalid version",
emulationVerson: "1.foo",
parseError: "illegal version string",
name: "invalid version",
emulationVersion: "1.foo",
parseError: "illegal version string",
},
}
for i, test := range tests {
@ -162,7 +162,7 @@ func TestEffectiveVersionsFlag(t *testing.T) {
effective := NewEffectiveVersion("1.30")
effective.AddFlags(fs, "test")
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)})
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)})
if test.parseError != "" {
if !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err)