From f5af0796fc67c0d1c7524b603307f094a65e65b5 Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Thu, 21 Dec 2017 11:27:20 +0800 Subject: [PATCH] pass APIEnablement through apiserver chain Kubernetes-commit: 2f403b7ad18a179514f1de77e29f1a2549ef030a --- pkg/server/config.go | 6 + pkg/server/options/api_enablement.go | 111 ++++++++++++++ pkg/server/resourceconfig/doc.go | 18 +++ pkg/server/resourceconfig/helpers.go | 164 ++++++++++++++++++++ pkg/server/resourceconfig/helpers_test.go | 178 ++++++++++++++++++++++ pkg/server/storage/storage_factory.go | 9 +- 6 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 pkg/server/options/api_enablement.go create mode 100644 pkg/server/resourceconfig/doc.go create mode 100644 pkg/server/resourceconfig/helpers.go create mode 100644 pkg/server/resourceconfig/helpers_test.go diff --git a/pkg/server/config.go b/pkg/server/config.go index 35acaa812..8ad2ddb5a 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -57,6 +57,7 @@ import ( genericfilters "k8s.io/apiserver/pkg/server/filters" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" + serverstore "k8s.io/apiserver/pkg/server/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" @@ -175,6 +176,11 @@ type Config struct { // if the client requests it via Accept-Encoding EnableAPIResponseCompression bool + // MergedResourceConfig indicates which groupVersion enabled and its resources enabled/disabled. + // This is composed of genericapiserver defaultAPIResourceConfig and those parsed from flags. + // If not specify any in flags, then genericapiserver will only enable defaultAPIResourceConfig. + MergedResourceConfig *serverstore.ResourceConfig + //=========================================================================== // values below here are targets for removal //=========================================================================== diff --git a/pkg/server/options/api_enablement.go b/pkg/server/options/api_enablement.go new file mode 100644 index 000000000..3901511f6 --- /dev/null +++ b/pkg/server/options/api_enablement.go @@ -0,0 +1,111 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/resourceconfig" + serverstore "k8s.io/apiserver/pkg/server/storage" + utilflag "k8s.io/apiserver/pkg/util/flag" +) + +// APIEnablementOptions contains the options for which resources to turn on and off. +// Given small aggregated API servers, this option isn't required for "normal" API servers +type APIEnablementOptions struct { + RuntimeConfig utilflag.ConfigurationMap +} + +func NewAPIEnablementOptions() *APIEnablementOptions { + return &APIEnablementOptions{ + RuntimeConfig: make(utilflag.ConfigurationMap), + } +} + +// AddFlags adds flags for a specific APIServer to the specified FlagSet +func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) { + fs.Var(&s.RuntimeConfig, "runtime-config", ""+ + "A set of key=value pairs that describe runtime configuration that may be passed "+ + "to apiserver. / (or for the core group) key can be used to "+ + "turn on/off specific api versions. api/all is special key to control all api versions, "+ + "be careful setting it false, unless you know what you do. api/legacy is deprecated, "+ + "we will remove it in the future, so stop using it.") +} + +// Validate validates RuntimeConfig with a list of registries. +// Usually this list only has one element, the apiserver registry of the process. +// But in the advanced (and usually not recommended) case of delegated apiservers there can be more. +// Validate will filter out the known groups of each registry. +// If anything is left over after that, an error is returned. +func (s *APIEnablementOptions) Validate(registries ...GroupRegisty) []error { + if s == nil { + return nil + } + + errors := []error{} + if s.RuntimeConfig["api/all"] == "false" && len(s.RuntimeConfig) == 1 { + // Do not allow only set api/all=false, in such case apiserver startup has no meaning. + return append(errors, fmt.Errorf("invliad key with only api/all=false")) + } + + groups, err := resourceconfig.ParseGroups(s.RuntimeConfig) + if err != nil { + return append(errors, err) + } + + for _, registry := range registries { + // filter out known groups + groups = unknownGroups(groups, registry) + } + if len(groups) != 0 { + errors = append(errors, fmt.Errorf("unknown api groups %s", strings.Join(groups, ","))) + } + + return errors +} + +// ApplyTo override MergedResourceConfig with defaults and registry +func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error { + if s == nil { + return nil + } + + mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(defaultResourceConfig, s.RuntimeConfig, registry) + c.MergedResourceConfig = mergedResourceConfig + + return err +} + +func unknownGroups(groups []string, registry GroupRegisty) []string { + unknownGroups := []string{} + for _, group := range groups { + if !registry.IsRegistered(group) { + unknownGroups = append(unknownGroups, group) + } + } + return unknownGroups +} + +// GroupRegisty provides a method to check whether given group is registered. +type GroupRegisty interface { + // IsRegistered returns true if given group is registered. + IsRegistered(group string) bool +} diff --git a/pkg/server/resourceconfig/doc.go b/pkg/server/resourceconfig/doc.go new file mode 100644 index 000000000..0dae21535 --- /dev/null +++ b/pkg/server/resourceconfig/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resourceconfig contains the resource config related helper functions. +package resourceconfig // import "k8s.io/apiserver/pkg/server/resourceconfig" diff --git a/pkg/server/resourceconfig/helpers.go b/pkg/server/resourceconfig/helpers.go new file mode 100644 index 000000000..3ac79b74a --- /dev/null +++ b/pkg/server/resourceconfig/helpers.go @@ -0,0 +1,164 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourceconfig + +import ( + "fmt" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + serverstore "k8s.io/apiserver/pkg/server/storage" + utilflag "k8s.io/apiserver/pkg/util/flag" +) + +// GroupVersionRegistry provides access to registered group versions. +type GroupVersionRegistry interface { + // IsRegistered returns true if given group is registered. + IsRegistered(group string) bool + // IsRegisteredVersion returns true if given version is registered. + IsRegisteredVersion(v schema.GroupVersion) bool + // RegisteredGroupVersions returns all registered group versions. + RegisteredGroupVersions() []schema.GroupVersion +} + +// MergeResourceEncodingConfigs merges the given defaultResourceConfig with specific GroupVersionResource overrides. +func MergeResourceEncodingConfigs( + defaultResourceEncoding *serverstore.DefaultResourceEncodingConfig, + resourceEncodingOverrides []schema.GroupVersionResource, +) *serverstore.DefaultResourceEncodingConfig { + resourceEncodingConfig := defaultResourceEncoding + for _, gvr := range resourceEncodingOverrides { + resourceEncodingConfig.SetResourceEncoding(gvr.GroupResource(), gvr.GroupVersion(), + schema.GroupVersion{Group: gvr.Group, Version: runtime.APIVersionInternal}) + } + return resourceEncodingConfig +} + +// MergeGroupEncodingConfigs merges the given defaultResourceConfig with specific GroupVersion overrides. +func MergeGroupEncodingConfigs( + defaultResourceEncoding *serverstore.DefaultResourceEncodingConfig, + storageEncodingOverrides map[string]schema.GroupVersion, +) *serverstore.DefaultResourceEncodingConfig { + resourceEncodingConfig := defaultResourceEncoding + for group, storageEncodingVersion := range storageEncodingOverrides { + resourceEncodingConfig.SetVersionEncoding(group, storageEncodingVersion, schema.GroupVersion{Group: group, Version: runtime.APIVersionInternal}) + } + return resourceEncodingConfig +} + +// MergeAPIResourceConfigs merges the given defaultAPIResourceConfig with the given resourceConfigOverrides. +// Exclude the groups not registered in registry, and check if version is +// not registered in group, then it will fail. +func MergeAPIResourceConfigs( + defaultAPIResourceConfig *serverstore.ResourceConfig, + resourceConfigOverrides utilflag.ConfigurationMap, + registry GroupVersionRegistry, +) (*serverstore.ResourceConfig, error) { + resourceConfig := defaultAPIResourceConfig + overrides := resourceConfigOverrides + + // "api/all=false" allows users to selectively enable specific api versions. + allAPIFlagValue, ok := overrides["api/all"] + if ok { + if allAPIFlagValue == "false" { + // Disable all group versions. + resourceConfig.DisableVersions(registry.RegisteredGroupVersions()...) + } else if allAPIFlagValue == "true" { + resourceConfig.EnableVersions(registry.RegisteredGroupVersions()...) + } + } + + // "={true|false} allows users to enable/disable API. + // This takes preference over api/all, if specified. + // Iterate through all group/version overrides specified in runtimeConfig. + for key := range overrides { + // Have already handled them above. Can skip them here. + if key == "api/all" { + continue + } + + tokens := strings.Split(key, "/") + if len(tokens) != 2 { + continue + } + groupVersionString := tokens[0] + "/" + tokens[1] + groupVersion, err := schema.ParseGroupVersion(groupVersionString) + if err != nil { + return nil, fmt.Errorf("invalid key %s", key) + } + + // Exclude group not registered into the registry. + if !registry.IsRegistered(groupVersion.Group) { + continue + } + + // Verify that the groupVersion is registered into registry. + if !registry.IsRegisteredVersion(groupVersion) { + return nil, fmt.Errorf("group version %s that has not been registered", groupVersion.String()) + } + enabled, err := getRuntimeConfigValue(overrides, key, false) + if err != nil { + return nil, err + } + if enabled { + resourceConfig.EnableVersions(groupVersion) + } else { + resourceConfig.DisableVersions(groupVersion) + } + } + + return resourceConfig, nil +} + +func getRuntimeConfigValue(overrides utilflag.ConfigurationMap, apiKey string, defaultValue bool) (bool, error) { + flagValue, ok := overrides[apiKey] + if ok { + if flagValue == "" { + return true, nil + } + boolValue, err := strconv.ParseBool(flagValue) + if err != nil { + return false, fmt.Errorf("invalid value of %s: %s, err: %v", apiKey, flagValue, err) + } + return boolValue, nil + } + return defaultValue, nil +} + +// ParseGroups takes in resourceConfig and returns parsed groups. +func ParseGroups(resourceConfig utilflag.ConfigurationMap) ([]string, error) { + groups := []string{} + for key := range resourceConfig { + if key == "api/all" { + continue + } + tokens := strings.Split(key, "/") + if len(tokens) != 2 && len(tokens) != 3 { + return groups, fmt.Errorf("runtime-config invalid key %s", key) + } + groupVersionString := tokens[0] + "/" + tokens[1] + groupVersion, err := schema.ParseGroupVersion(groupVersionString) + if err != nil { + return nil, fmt.Errorf("runtime-config invalid key %s", key) + } + groups = append(groups, groupVersion.Group) + } + + return groups, nil +} diff --git a/pkg/server/resourceconfig/helpers_test.go b/pkg/server/resourceconfig/helpers_test.go new file mode 100644 index 000000000..801d78414 --- /dev/null +++ b/pkg/server/resourceconfig/helpers_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourceconfig + +import ( + "reflect" + "testing" + + apiv1 "k8s.io/api/core/v1" + extensionsapiv1beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/apimachinery" + "k8s.io/apimachinery/pkg/apimachinery/registered" + "k8s.io/apimachinery/pkg/runtime/schema" + serverstore "k8s.io/apiserver/pkg/server/storage" +) + +func TestParseRuntimeConfig(t *testing.T) { + registry := newFakeRegistry() + apiv1GroupVersion := apiv1.SchemeGroupVersion + testCases := []struct { + runtimeConfig map[string]string + defaultResourceConfig func() *serverstore.ResourceConfig + expectedAPIConfig func() *serverstore.ResourceConfig + err bool + }{ + { + // everything default value. + runtimeConfig: map[string]string{}, + defaultResourceConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + err: false, + }, + { + // no runtimeConfig override. + runtimeConfig: map[string]string{}, + defaultResourceConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + config.DisableVersions(extensionsapiv1beta1.SchemeGroupVersion) + return config + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + config.DisableVersions(extensionsapiv1beta1.SchemeGroupVersion) + return config + }, + err: false, + }, + { + // version enabled by runtimeConfig override. + runtimeConfig: map[string]string{ + "extensions/v1beta1": "", + }, + defaultResourceConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + return config + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + return config + }, + err: false, + }, + { + // Disable v1. + runtimeConfig: map[string]string{ + "/v1": "false", + }, + defaultResourceConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + config.DisableVersions(apiv1GroupVersion) + return config + }, + err: false, + }, + { + // invalid runtime config + runtimeConfig: map[string]string{ + "invalidgroup/version": "false", + }, + defaultResourceConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + err: false, + }, + { + // enable all + runtimeConfig: map[string]string{ + "api/all": "true", + }, + defaultResourceConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + config.EnableVersions(registry.RegisteredGroupVersions()...) + return config + }, + err: false, + }, + { + // only enable v1 + runtimeConfig: map[string]string{ + "api/all": "false", + "/v1": "true", + }, + defaultResourceConfig: func() *serverstore.ResourceConfig { + return newFakeAPIResourceConfigSource() + }, + expectedAPIConfig: func() *serverstore.ResourceConfig { + config := newFakeAPIResourceConfigSource() + config.DisableVersions(extensionsapiv1beta1.SchemeGroupVersion) + return config + }, + err: false, + }, + } + for index, test := range testCases { + actualDisablers, err := MergeAPIResourceConfigs(test.defaultResourceConfig(), test.runtimeConfig, registry) + if err == nil && test.err { + t.Fatalf("expected error for test case: %v", index) + } else if err != nil && !test.err { + t.Fatalf("unexpected error: %s, for test: %v", err, test) + } + + expectedConfig := test.expectedAPIConfig() + if err == nil && !reflect.DeepEqual(actualDisablers, expectedConfig) { + t.Fatalf("%v: unexpected apiResourceDisablers. Actual: %v\n expected: %v", test.runtimeConfig, actualDisablers, expectedConfig) + } + } +} + +func newFakeAPIResourceConfigSource() *serverstore.ResourceConfig { + ret := serverstore.NewResourceConfig() + // NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list. + ret.EnableVersions( + apiv1.SchemeGroupVersion, + extensionsapiv1beta1.SchemeGroupVersion, + ) + + return ret +} + +func newFakeRegistry() *registered.APIRegistrationManager { + registry := registered.NewOrDie("") + + registry.RegisterGroup(apimachinery.GroupMeta{ + GroupVersion: apiv1.SchemeGroupVersion, + }) + registry.RegisterGroup(apimachinery.GroupMeta{ + GroupVersion: extensionsapiv1beta1.SchemeGroupVersion, + }) + registry.RegisterVersions([]schema.GroupVersion{apiv1.SchemeGroupVersion, extensionsapiv1beta1.SchemeGroupVersion}) + return registry +} diff --git a/pkg/server/storage/storage_factory.go b/pkg/server/storage/storage_factory.go index 429e14d5c..9e56eedc2 100644 --- a/pkg/server/storage/storage_factory.go +++ b/pkg/server/storage/storage_factory.go @@ -151,7 +151,14 @@ var _ StorageFactory = &DefaultStorageFactory{} const AllResources = "*" -func NewDefaultStorageFactory(config storagebackend.Config, defaultMediaType string, defaultSerializer runtime.StorageSerializer, resourceEncodingConfig ResourceEncodingConfig, resourceConfig APIResourceConfigSource, specialDefaultResourcePrefixes map[schema.GroupResource]string) *DefaultStorageFactory { +func NewDefaultStorageFactory( + config storagebackend.Config, + defaultMediaType string, + defaultSerializer runtime.StorageSerializer, + resourceEncodingConfig ResourceEncodingConfig, + resourceConfig APIResourceConfigSource, + specialDefaultResourcePrefixes map[schema.GroupResource]string, +) *DefaultStorageFactory { config.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) if len(defaultMediaType) == 0 { defaultMediaType = runtime.ContentTypeJSON