diff --git a/pkg/admission/initializer/interfaces.go b/pkg/admission/initializer/interfaces.go index 2b3031aa2..86a6df1c2 100644 --- a/pkg/admission/initializer/interfaces.go +++ b/pkg/admission/initializer/interfaces.go @@ -19,6 +19,7 @@ package initializer import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" + quota "k8s.io/apiserver/pkg/quota/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/component-base/featuregate" @@ -42,6 +43,12 @@ type WantsAuthorizer interface { admission.InitializationValidator } +// WantsQuotaConfiguration defines a function which sets quota configuration for admission plugins that need it. +type WantsQuotaConfiguration interface { + SetQuotaConfiguration(quota.Configuration) + admission.InitializationValidator +} + // WantsFeatureGate defines a function which passes the featureGates for inspection by an admission plugin. // Admission plugins should not hold a reference to the featureGates. Instead, they should query a particular one // and assign it to a simple bool in the admission plugin struct. diff --git a/pkg/admission/plugin/resourcequota/admission.go b/pkg/admission/plugin/resourcequota/admission.go new file mode 100644 index 000000000..f2a0f8d0b --- /dev/null +++ b/pkg/admission/plugin/resourcequota/admission.go @@ -0,0 +1,144 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "context" + "fmt" + "io" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apiserver/pkg/admission" + genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer" + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/validation" + quota "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/apiserver/pkg/quota/v1/generic" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" +) + +// PluginName is a string with the name of the plugin +const PluginName = "ResourceQuota" + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, + func(config io.Reader) (admission.Interface, error) { + // load the configuration provided (if any) + configuration, err := LoadConfiguration(config) + if err != nil { + return nil, err + } + // validate the configuration (if any) + if configuration != nil { + if errs := validation.ValidateConfiguration(configuration); len(errs) != 0 { + return nil, errs.ToAggregate() + } + } + return NewResourceQuota(configuration, 5, make(chan struct{})) + }) +} + +// QuotaAdmission implements an admission controller that can enforce quota constraints +type QuotaAdmission struct { + *admission.Handler + config *resourcequotaapi.Configuration + stopCh <-chan struct{} + quotaConfiguration quota.Configuration + numEvaluators int + quotaAccessor *quotaAccessor + evaluator Evaluator +} + +var _ admission.ValidationInterface = &QuotaAdmission{} +var _ = genericadmissioninitializer.WantsExternalKubeInformerFactory(&QuotaAdmission{}) +var _ = genericadmissioninitializer.WantsExternalKubeClientSet(&QuotaAdmission{}) +var _ = genericadmissioninitializer.WantsQuotaConfiguration(&QuotaAdmission{}) + +type liveLookupEntry struct { + expiry time.Time + items []*corev1.ResourceQuota +} + +// NewResourceQuota configures an admission controller that can enforce quota constraints +// using the provided registry. The registry must have the capability to handle group/kinds that +// are persisted by the server this admission controller is intercepting +func NewResourceQuota(config *resourcequotaapi.Configuration, numEvaluators int, stopCh <-chan struct{}) (*QuotaAdmission, error) { + quotaAccessor, err := newQuotaAccessor() + if err != nil { + return nil, err + } + + return &QuotaAdmission{ + Handler: admission.NewHandler(admission.Create, admission.Update), + stopCh: stopCh, + numEvaluators: numEvaluators, + config: config, + quotaAccessor: quotaAccessor, + }, nil +} + +// SetExternalKubeClientSet registers the client into QuotaAdmission +func (a *QuotaAdmission) SetExternalKubeClientSet(client kubernetes.Interface) { + a.quotaAccessor.client = client +} + +// SetExternalKubeInformerFactory registers an informer factory into QuotaAdmission +func (a *QuotaAdmission) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { + a.quotaAccessor.lister = f.Core().V1().ResourceQuotas().Lister() +} + +// SetQuotaConfiguration assigns and initializes configuration and evaluator for QuotaAdmission +func (a *QuotaAdmission) SetQuotaConfiguration(c quota.Configuration) { + a.quotaConfiguration = c + a.evaluator = NewQuotaEvaluator(a.quotaAccessor, a.quotaConfiguration.IgnoredResources(), generic.NewRegistry(a.quotaConfiguration.Evaluators()), nil, a.config, a.numEvaluators, a.stopCh) +} + +// ValidateInitialization ensures an authorizer is set. +func (a *QuotaAdmission) ValidateInitialization() error { + if a.quotaAccessor == nil { + return fmt.Errorf("missing quotaAccessor") + } + if a.quotaAccessor.client == nil { + return fmt.Errorf("missing quotaAccessor.client") + } + if a.quotaAccessor.lister == nil { + return fmt.Errorf("missing quotaAccessor.lister") + } + if a.quotaConfiguration == nil { + return fmt.Errorf("missing quotaConfiguration") + } + if a.evaluator == nil { + return fmt.Errorf("missing evaluator") + } + return nil +} + +// Validate makes admission decisions while enforcing quota +func (a *QuotaAdmission) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) { + // ignore all operations that correspond to sub-resource actions + if attr.GetSubresource() != "" { + return nil + } + // ignore all operations that are not namespaced + if attr.GetNamespace() == "" { + return nil + } + return a.evaluator.Evaluate(attr) +} diff --git a/pkg/admission/plugin/resourcequota/admission_test.go b/pkg/admission/plugin/resourcequota/admission_test.go new file mode 100644 index 000000000..461eb9bb7 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/admission_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestPrettyPrint(t *testing.T) { + toResourceList := func(resources map[corev1.ResourceName]string) corev1.ResourceList { + resourceList := corev1.ResourceList{} + for key, value := range resources { + resourceList[key] = resource.MustParse(value) + } + return resourceList + } + testCases := []struct { + input corev1.ResourceList + expected string + }{ + { + input: toResourceList(map[corev1.ResourceName]string{ + corev1.ResourceCPU: "100m", + }), + expected: "cpu=100m", + }, + { + input: toResourceList(map[corev1.ResourceName]string{ + corev1.ResourcePods: "10", + corev1.ResourceServices: "10", + corev1.ResourceReplicationControllers: "10", + corev1.ResourceServicesNodePorts: "10", + corev1.ResourceRequestsCPU: "100m", + corev1.ResourceRequestsMemory: "100Mi", + corev1.ResourceLimitsCPU: "100m", + corev1.ResourceLimitsMemory: "100Mi", + }), + expected: "limits.cpu=100m,limits.memory=100Mi,pods=10,replicationcontrollers=10,requests.cpu=100m,requests.memory=100Mi,services=10,services.nodeports=10", + }, + } + for i, testCase := range testCases { + result := prettyPrint(testCase.input) + if result != testCase.expected { + t.Errorf("Pretty print did not give stable sorted output[%d], expected %v, but got %v", i, testCase.expected, result) + } + } +} + +func TestHasUsageStats(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceQuota + relevant []corev1.ResourceName + expected bool + }{ + "empty": { + a: corev1.ResourceQuota{Status: corev1.ResourceQuotaStatus{Hard: corev1.ResourceList{}}}, + relevant: []corev1.ResourceName{corev1.ResourceMemory}, + expected: true, + }, + "hard-only": { + a: corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Used: corev1.ResourceList{}, + }, + }, + relevant: []corev1.ResourceName{corev1.ResourceMemory}, + expected: false, + }, + "hard-used": { + a: corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + relevant: []corev1.ResourceName{corev1.ResourceMemory}, + expected: true, + }, + "hard-used-relevant": { + a: corev1.ResourceQuota{ + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("1"), + }, + Used: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + relevant: []corev1.ResourceName{corev1.ResourceMemory}, + expected: true, + }, + } + for testName, testCase := range testCases { + if result := hasUsageStats(&testCase.a, testCase.relevant); result != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, result) + } + } +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/OWNERS b/pkg/admission/plugin/resourcequota/apis/resourcequota/OWNERS new file mode 100644 index 000000000..cc22b2227 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/OWNERS @@ -0,0 +1,9 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: +- deads2k +- derekwaynecarr +approvers: +- deads2k +- derekwaynecarr +- smarterclayton \ No newline at end of file diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/doc.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/doc.go new file mode 100644 index 000000000..b8c0ea047 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +package resourcequota // import "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/install/install.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/install/install.go new file mode 100644 index 000000000..6eabe1809 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/install/install.go @@ -0,0 +1,42 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package install installs the experimental API group, making it available as +// an option to all of the API encoding/decoding machinery. +package install + +import ( + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + resourcequotav1 "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1" + resourcequotav1alpha1 "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1" + resourcequotav1beta1 "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1" +) + +// Install registers the API group and adds types to a scheme +func Install(scheme *runtime.Scheme) { + utilruntime.Must(resourcequotaapi.AddToScheme(scheme)) + + // v1beta1 and v1alpha1 are in the k8s.io-suffixed group + utilruntime.Must(resourcequotav1beta1.AddToScheme(scheme)) + utilruntime.Must(resourcequotav1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(resourcequotav1beta1.SchemeGroupVersion, resourcequotav1alpha1.SchemeGroupVersion)) + + // v1 is in the config.k8s.io-suffixed group + utilruntime.Must(resourcequotav1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(resourcequotav1.SchemeGroupVersion)) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/register.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/register.go new file mode 100644 index 000000000..4d9735d49 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/register.go @@ -0,0 +1,51 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // SchemeBuilder is a pointer used to call AddToScheme + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme is used to register the types to API encoding/decoding machinery + AddToScheme = SchemeBuilder.AddToScheme +) + +// LegacyGroupName is the group name use in this package +const LegacyGroupName = "resourcequota.admission.k8s.io" + +// LegacySchemeGroupVersion is group version used to register these objects +var LegacySchemeGroupVersion = schema.GroupVersion{Group: LegacyGroupName, Version: runtime.APIVersionInternal} + +// GroupName is the group name use in this package +const GroupName = "apiserver.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(LegacySchemeGroupVersion, + &Configuration{}, + ) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ResourceQuotaConfiguration"), + &Configuration{}, + ) + return nil +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/types.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/types.go new file mode 100644 index 000000000..b8ffc1042 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/types.go @@ -0,0 +1,72 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Configuration provides configuration for the ResourceQuota admission controller. +type Configuration struct { + metav1.TypeMeta + + // LimitedResources whose consumption is limited by default. + // +optional + LimitedResources []LimitedResource +} + +// LimitedResource matches a resource whose consumption is limited by default. +// To consume the resource, there must exist an associated quota that limits +// its consumption. +type LimitedResource struct { + + // APIGroup is the name of the APIGroup that contains the limited resource. + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // Resource is the name of the resource this rule applies to. + // For example, if the administrator wants to limit consumption + // of a storage resource associated with persistent volume claims, + // the value would be "persistentvolumeclaims". + Resource string `json:"resource"` + + // For each intercepted request, the quota system will evaluate + // its resource usage. It will iterate through each resource consumed + // and if the resource contains any substring in this listing, the + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // that a quota must exist to consume persistent volume claims associated + // with any storage class, the list would include + // ".storageclass.storage.k8s.io/requests.storage" + MatchContains []string + + // For each intercepted request, the quota system will figure out if the input object + // satisfies a scope which is present in this listing, then + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // a quota must exist to create a pod with "cluster-services" priorityclass + // the list would include + // "PriorityClassNameIn=cluster-services" + // +optional + // MatchScopes []string `json:"matchScopes,omitempty"` + MatchScopes []corev1.ScopedResourceSelectorRequirement `json:"matchScopes,omitempty"` +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/defaults.go new file mode 100644 index 000000000..1a2f1b444 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/defaults.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import kruntime "k8s.io/apimachinery/pkg/runtime" + +func addDefaultingFuncs(scheme *kruntime.Scheme) error { + return RegisterDefaults(scheme) +} + +func SetDefaults_Configuration(obj *Configuration) {} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/doc.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/doc.go new file mode 100644 index 000000000..4eaeff77a --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:conversion-gen=k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota +// +k8s:defaulter-gen=TypeMeta +// +groupName=resourcequota.admission.k8s.io + +// Package v1 is the v1 version of the API. +package v1 // import "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1" diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/register.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/register.go new file mode 100644 index 000000000..04969a162 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/register.go @@ -0,0 +1,51 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package +const GroupName = "apiserver.config.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} + +var ( + // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. + // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. + + // SchemeBuilder is a pointer used to call AddToScheme + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme is used to register the types to API encoding/decoding machinery + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ResourceQuotaConfiguration"), &Configuration{}) + return nil +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/types.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/types.go new file mode 100644 index 000000000..e703724c6 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/types.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Configuration provides configuration for the ResourceQuota admission controller. +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // LimitedResources whose consumption is limited by default. + // +optional + LimitedResources []LimitedResource `json:"limitedResources"` +} + +// LimitedResource matches a resource whose consumption is limited by default. +// To consume the resource, there must exist an associated quota that limits +// its consumption. +type LimitedResource struct { + + // APIGroup is the name of the APIGroup that contains the limited resource. + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // Resource is the name of the resource this rule applies to. + // For example, if the administrator wants to limit consumption + // of a storage resource associated with persistent volume claims, + // the value would be "persistentvolumeclaims". + Resource string `json:"resource"` + + // For each intercepted request, the quota system will evaluate + // its resource usage. It will iterate through each resource consumed + // and if the resource contains any substring in this listing, the + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // that a quota must exist to consume persistent volume claims associated + // with any storage class, the list would include + // ".storageclass.storage.k8s.io/requests.storage" + MatchContains []string `json:"matchContains,omitempty"` + // For each intercepted request, the quota system will figure out if the input object + // satisfies a scope which is present in this listing, then + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // a quota must exist to create a pod with "cluster-services" priorityclass + // the list would include "scopeName=PriorityClass, Operator=In, Value=cluster-services" + // +optional + MatchScopes []v1.ScopedResourceSelectorRequirement `json:"matchScopes,omitempty"` +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.conversion.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.conversion.go new file mode 100644 index 000000000..e1fca3bd3 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.conversion.go @@ -0,0 +1,106 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by conversion-gen. DO NOT EDIT. + +package v1 + +import ( + unsafe "unsafe" + + corev1 "k8s.io/api/core/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + resourcequota "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*resourcequota.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_Configuration_To_resourcequota_Configuration(a.(*Configuration), b.(*resourcequota.Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_Configuration_To_v1_Configuration(a.(*resourcequota.Configuration), b.(*Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LimitedResource)(nil), (*resourcequota.LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_LimitedResource_To_resourcequota_LimitedResource(a.(*LimitedResource), b.(*resourcequota.LimitedResource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.LimitedResource)(nil), (*LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_LimitedResource_To_v1_LimitedResource(a.(*resourcequota.LimitedResource), b.(*LimitedResource), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]resourcequota.LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_v1_Configuration_To_resourcequota_Configuration is an autogenerated conversion function. +func Convert_v1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + return autoConvert_v1_Configuration_To_resourcequota_Configuration(in, out, s) +} + +func autoConvert_resourcequota_Configuration_To_v1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_resourcequota_Configuration_To_v1_Configuration is an autogenerated conversion function. +func Convert_resourcequota_Configuration_To_v1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + return autoConvert_resourcequota_Configuration_To_v1_Configuration(in, out, s) +} + +func autoConvert_v1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]corev1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_v1_LimitedResource_To_resourcequota_LimitedResource is an autogenerated conversion function. +func Convert_v1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + return autoConvert_v1_LimitedResource_To_resourcequota_LimitedResource(in, out, s) +} + +func autoConvert_resourcequota_LimitedResource_To_v1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]corev1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_resourcequota_LimitedResource_To_v1_LimitedResource is an autogenerated conversion function. +func Convert_resourcequota_LimitedResource_To_v1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + return autoConvert_resourcequota_LimitedResource_To_v1_LimitedResource(in, out, s) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.deepcopy.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000..f777dbb0e --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.deepcopy.go @@ -0,0 +1,86 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.LimitedResources != nil { + in, out := &in.LimitedResources, &out.LimitedResources + *out = make([]LimitedResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitedResource) DeepCopyInto(out *LimitedResource) { + *out = *in + if in.MatchContains != nil { + in, out := &in.MatchContains, &out.MatchContains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchScopes != nil { + in, out := &in.MatchScopes, &out.MatchScopes + *out = make([]corev1.ScopedResourceSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitedResource. +func (in *LimitedResource) DeepCopy() *LimitedResource { + if in == nil { + return nil + } + out := new(LimitedResource) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.defaults.go new file mode 100644 index 000000000..329955354 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1/zz_generated.defaults.go @@ -0,0 +1,37 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/defaults.go new file mode 100644 index 000000000..ebade2de2 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/defaults.go @@ -0,0 +1,25 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import kruntime "k8s.io/apimachinery/pkg/runtime" + +func addDefaultingFuncs(scheme *kruntime.Scheme) error { + return RegisterDefaults(scheme) +} + +func SetDefaults_Configuration(obj *Configuration) {} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/doc.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/doc.go new file mode 100644 index 000000000..b1c83e40d --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:conversion-gen=k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota +// +k8s:defaulter-gen=TypeMeta +// +groupName=resourcequota.admission.k8s.io + +// Package v1alpha1 is the v1alpha1 version of the API. +package v1alpha1 // import "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1" diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/register.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/register.go new file mode 100644 index 000000000..df604f689 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/register.go @@ -0,0 +1,53 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package +const GroupName = "resourcequota.admission.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + +var ( + // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. + // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. + + // SchemeBuilder is a pointer used to call AddToScheme + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme is used to register the types to API encoding/decoding machinery + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &Configuration{}, + ) + return nil +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/types.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/types.go new file mode 100644 index 000000000..9ea8fe5ad --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/types.go @@ -0,0 +1,69 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Configuration provides configuration for the ResourceQuota admission controller. +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // LimitedResources whose consumption is limited by default. + // +optional + LimitedResources []LimitedResource `json:"limitedResources"` +} + +// LimitedResource matches a resource whose consumption is limited by default. +// To consume the resource, there must exist an associated quota that limits +// its consumption. +type LimitedResource struct { + + // APIGroup is the name of the APIGroup that contains the limited resource. + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // Resource is the name of the resource this rule applies to. + // For example, if the administrator wants to limit consumption + // of a storage resource associated with persistent volume claims, + // the value would be "persistentvolumeclaims". + Resource string `json:"resource"` + + // For each intercepted request, the quota system will evaluate + // its resource usage. It will iterate through each resource consumed + // and if the resource contains any substring in this listing, the + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // that a quota must exist to consume persistent volume claims associated + // with any storage class, the list would include + // ".storageclass.storage.k8s.io/requests.storage" + MatchContains []string `json:"matchContains,omitempty"` + // For each intercepted request, the quota system will figure out if the input object + // satisfies a scope which is present in this listing, then + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // a quota must exist to create a pod with "cluster-services" priorityclass + // the list would include "scopeName=PriorityClass, Operator=In, Value=cluster-services" + // +optional + MatchScopes []v1.ScopedResourceSelectorRequirement `json:"matchScopes,omitempty"` +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go new file mode 100644 index 000000000..8daa12683 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go @@ -0,0 +1,106 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by conversion-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + unsafe "unsafe" + + v1 "k8s.io/api/core/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + resourcequota "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*resourcequota.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_Configuration_To_resourcequota_Configuration(a.(*Configuration), b.(*resourcequota.Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_Configuration_To_v1alpha1_Configuration(a.(*resourcequota.Configuration), b.(*Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LimitedResource)(nil), (*resourcequota.LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_LimitedResource_To_resourcequota_LimitedResource(a.(*LimitedResource), b.(*resourcequota.LimitedResource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.LimitedResource)(nil), (*LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_LimitedResource_To_v1alpha1_LimitedResource(a.(*resourcequota.LimitedResource), b.(*LimitedResource), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1alpha1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]resourcequota.LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_v1alpha1_Configuration_To_resourcequota_Configuration is an autogenerated conversion function. +func Convert_v1alpha1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + return autoConvert_v1alpha1_Configuration_To_resourcequota_Configuration(in, out, s) +} + +func autoConvert_resourcequota_Configuration_To_v1alpha1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_resourcequota_Configuration_To_v1alpha1_Configuration is an autogenerated conversion function. +func Convert_resourcequota_Configuration_To_v1alpha1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + return autoConvert_resourcequota_Configuration_To_v1alpha1_Configuration(in, out, s) +} + +func autoConvert_v1alpha1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]v1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_v1alpha1_LimitedResource_To_resourcequota_LimitedResource is an autogenerated conversion function. +func Convert_v1alpha1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + return autoConvert_v1alpha1_LimitedResource_To_resourcequota_LimitedResource(in, out, s) +} + +func autoConvert_resourcequota_LimitedResource_To_v1alpha1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]v1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_resourcequota_LimitedResource_To_v1alpha1_LimitedResource is an autogenerated conversion function. +func Convert_resourcequota_LimitedResource_To_v1alpha1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + return autoConvert_resourcequota_LimitedResource_To_v1alpha1_LimitedResource(in, out, s) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.deepcopy.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..95a100b92 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,86 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.LimitedResources != nil { + in, out := &in.LimitedResources, &out.LimitedResources + *out = make([]LimitedResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitedResource) DeepCopyInto(out *LimitedResource) { + *out = *in + if in.MatchContains != nil { + in, out := &in.MatchContains, &out.MatchContains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchScopes != nil { + in, out := &in.MatchScopes, &out.MatchScopes + *out = make([]v1.ScopedResourceSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitedResource. +func (in *LimitedResource) DeepCopy() *LimitedResource { + if in == nil { + return nil + } + out := new(LimitedResource) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.defaults.go new file mode 100644 index 000000000..fbf617bbf --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1alpha1/zz_generated.defaults.go @@ -0,0 +1,37 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/defaults.go new file mode 100644 index 000000000..c56915a58 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/defaults.go @@ -0,0 +1,25 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import kruntime "k8s.io/apimachinery/pkg/runtime" + +func addDefaultingFuncs(scheme *kruntime.Scheme) error { + return RegisterDefaults(scheme) +} + +func SetDefaults_Configuration(obj *Configuration) {} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/doc.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/doc.go new file mode 100644 index 000000000..6d94d774c --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:conversion-gen=k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota +// +k8s:defaulter-gen=TypeMeta +// +groupName=resourcequota.admission.k8s.io + +// Package v1beta1 is the v1beta1 version of the API. +package v1beta1 // import "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1" diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/register.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/register.go new file mode 100644 index 000000000..0bdbc14f0 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/register.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package +const GroupName = "resourcequota.admission.k8s.io" + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} + +var ( + // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. + // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. + + // SchemeBuilder is a pointer used to call AddToScheme + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme is used to register the types to API encoding/decoding machinery + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &Configuration{}, + ) + return nil +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/types.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/types.go new file mode 100644 index 000000000..2caf645f7 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/types.go @@ -0,0 +1,69 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Configuration provides configuration for the ResourceQuota admission controller. +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // LimitedResources whose consumption is limited by default. + // +optional + LimitedResources []LimitedResource `json:"limitedResources"` +} + +// LimitedResource matches a resource whose consumption is limited by default. +// To consume the resource, there must exist an associated quota that limits +// its consumption. +type LimitedResource struct { + + // APIGroup is the name of the APIGroup that contains the limited resource. + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // Resource is the name of the resource this rule applies to. + // For example, if the administrator wants to limit consumption + // of a storage resource associated with persistent volume claims, + // the value would be "persistentvolumeclaims". + Resource string `json:"resource"` + + // For each intercepted request, the quota system will evaluate + // its resource usage. It will iterate through each resource consumed + // and if the resource contains any substring in this listing, the + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // that a quota must exist to consume persistent volume claims associated + // with any storage class, the list would include + // ".storageclass.storage.k8s.io/requests.storage" + MatchContains []string `json:"matchContains,omitempty"` + // For each intercepted request, the quota system will figure out if the input object + // satisfies a scope which is present in this listing, then + // quota system will ensure that there is a covering quota. In the + // absence of a covering quota, the quota system will deny the request. + // For example, if an administrator wants to globally enforce that + // a quota must exist to create a pod with "cluster-services" priorityclass + // the list would include "scopeName=PriorityClass, Operator=In, Value=cluster-services" + // +optional + MatchScopes []v1.ScopedResourceSelectorRequirement `json:"matchScopes,omitempty"` +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.conversion.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.conversion.go new file mode 100644 index 000000000..250c3e158 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.conversion.go @@ -0,0 +1,106 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by conversion-gen. DO NOT EDIT. + +package v1beta1 + +import ( + unsafe "unsafe" + + v1 "k8s.io/api/core/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + resourcequota "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*resourcequota.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Configuration_To_resourcequota_Configuration(a.(*Configuration), b.(*resourcequota.Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_Configuration_To_v1beta1_Configuration(a.(*resourcequota.Configuration), b.(*Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LimitedResource)(nil), (*resourcequota.LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LimitedResource_To_resourcequota_LimitedResource(a.(*LimitedResource), b.(*resourcequota.LimitedResource), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*resourcequota.LimitedResource)(nil), (*LimitedResource)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_resourcequota_LimitedResource_To_v1beta1_LimitedResource(a.(*resourcequota.LimitedResource), b.(*LimitedResource), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1beta1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]resourcequota.LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_v1beta1_Configuration_To_resourcequota_Configuration is an autogenerated conversion function. +func Convert_v1beta1_Configuration_To_resourcequota_Configuration(in *Configuration, out *resourcequota.Configuration, s conversion.Scope) error { + return autoConvert_v1beta1_Configuration_To_resourcequota_Configuration(in, out, s) +} + +func autoConvert_resourcequota_Configuration_To_v1beta1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + out.LimitedResources = *(*[]LimitedResource)(unsafe.Pointer(&in.LimitedResources)) + return nil +} + +// Convert_resourcequota_Configuration_To_v1beta1_Configuration is an autogenerated conversion function. +func Convert_resourcequota_Configuration_To_v1beta1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { + return autoConvert_resourcequota_Configuration_To_v1beta1_Configuration(in, out, s) +} + +func autoConvert_v1beta1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]v1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_v1beta1_LimitedResource_To_resourcequota_LimitedResource is an autogenerated conversion function. +func Convert_v1beta1_LimitedResource_To_resourcequota_LimitedResource(in *LimitedResource, out *resourcequota.LimitedResource, s conversion.Scope) error { + return autoConvert_v1beta1_LimitedResource_To_resourcequota_LimitedResource(in, out, s) +} + +func autoConvert_resourcequota_LimitedResource_To_v1beta1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Resource = in.Resource + out.MatchContains = *(*[]string)(unsafe.Pointer(&in.MatchContains)) + out.MatchScopes = *(*[]v1.ScopedResourceSelectorRequirement)(unsafe.Pointer(&in.MatchScopes)) + return nil +} + +// Convert_resourcequota_LimitedResource_To_v1beta1_LimitedResource is an autogenerated conversion function. +func Convert_resourcequota_LimitedResource_To_v1beta1_LimitedResource(in *resourcequota.LimitedResource, out *LimitedResource, s conversion.Scope) error { + return autoConvert_resourcequota_LimitedResource_To_v1beta1_LimitedResource(in, out, s) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.deepcopy.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000..313439bc2 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,86 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.LimitedResources != nil { + in, out := &in.LimitedResources, &out.LimitedResources + *out = make([]LimitedResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitedResource) DeepCopyInto(out *LimitedResource) { + *out = *in + if in.MatchContains != nil { + in, out := &in.MatchContains, &out.MatchContains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchScopes != nil { + in, out := &in.MatchScopes, &out.MatchScopes + *out = make([]v1.ScopedResourceSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitedResource. +func (in *LimitedResource) DeepCopy() *LimitedResource { + if in == nil { + return nil + } + out := new(LimitedResource) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.defaults.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.defaults.go new file mode 100644 index 000000000..f64a3ac97 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/v1beta1/zz_generated.defaults.go @@ -0,0 +1,37 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1beta1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation.go new file mode 100644 index 000000000..4b23f0bca --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation.go @@ -0,0 +1,36 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// ValidateConfiguration validates the configuration. +func ValidateConfiguration(config *resourcequotaapi.Configuration) field.ErrorList { + allErrs := field.ErrorList{} + fldPath := field.NewPath("limitedResources") + for i, limitedResource := range config.LimitedResources { + idxPath := fldPath.Index(i) + if len(limitedResource.Resource) == 0 { + allErrs = append(allErrs, field.Required(idxPath.Child("resource"), "")) + } + } + return allErrs +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation_test.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation_test.go new file mode 100644 index 000000000..3e7c53c4d --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/validation/validation_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "testing" + + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" +) + +func TestValidateConfiguration(t *testing.T) { + successCases := []resourcequotaapi.Configuration{ + { + LimitedResources: []resourcequotaapi.LimitedResource{ + { + Resource: "pods", + MatchContains: []string{"requests.cpu"}, + }, + }, + }, + { + LimitedResources: []resourcequotaapi.LimitedResource{ + { + Resource: "persistentvolumeclaims", + MatchContains: []string{"requests.storage"}, + }, + }, + }, + } + for i := range successCases { + configuration := successCases[i] + if errs := ValidateConfiguration(&configuration); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + } + errorCases := map[string]resourcequotaapi.Configuration{ + "missing apiGroupResource": {LimitedResources: []resourcequotaapi.LimitedResource{ + {MatchContains: []string{"requests.cpu"}}, + }}, + } + for k, v := range errorCases { + if errs := ValidateConfiguration(&v); len(errs) == 0 { + t.Errorf("expected failure for %s", k) + } + } +} diff --git a/pkg/admission/plugin/resourcequota/apis/resourcequota/zz_generated.deepcopy.go b/pkg/admission/plugin/resourcequota/apis/resourcequota/zz_generated.deepcopy.go new file mode 100644 index 000000000..78baf66ef --- /dev/null +++ b/pkg/admission/plugin/resourcequota/apis/resourcequota/zz_generated.deepcopy.go @@ -0,0 +1,86 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package resourcequota + +import ( + v1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.LimitedResources != nil { + in, out := &in.LimitedResources, &out.LimitedResources + *out = make([]LimitedResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitedResource) DeepCopyInto(out *LimitedResource) { + *out = *in + if in.MatchContains != nil { + in, out := &in.MatchContains, &out.MatchContains + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchScopes != nil { + in, out := &in.MatchScopes, &out.MatchScopes + *out = make([]v1.ScopedResourceSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitedResource. +func (in *LimitedResource) DeepCopy() *LimitedResource { + if in == nil { + return nil + } + out := new(LimitedResource) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/admission/plugin/resourcequota/config.go b/pkg/admission/plugin/resourcequota/config.go new file mode 100644 index 000000000..84f03be25 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/config.go @@ -0,0 +1,67 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "fmt" + "io" + "io/ioutil" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/install" + resourcequotav1 "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota/v1" +) + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) +) + +func init() { + install.Install(scheme) +} + +// LoadConfiguration loads the provided configuration. +func LoadConfiguration(config io.Reader) (*resourcequotaapi.Configuration, error) { + // if no config is provided, return a default configuration + if config == nil { + externalConfig := &resourcequotav1.Configuration{} + scheme.Default(externalConfig) + internalConfig := &resourcequotaapi.Configuration{} + if err := scheme.Convert(externalConfig, internalConfig, nil); err != nil { + return nil, err + } + return internalConfig, nil + } + // we have a config so parse it. + data, err := ioutil.ReadAll(config) + if err != nil { + return nil, err + } + decoder := codecs.UniversalDecoder() + decodedObj, err := runtime.Decode(decoder, data) + if err != nil { + return nil, err + } + resourceQuotaConfiguration, ok := decodedObj.(*resourcequotaapi.Configuration) + if !ok { + return nil, fmt.Errorf("unexpected type: %T", decodedObj) + } + return resourceQuotaConfiguration, nil +} diff --git a/pkg/admission/plugin/resourcequota/config_test.go b/pkg/admission/plugin/resourcequota/config_test.go new file mode 100644 index 000000000..1b03c405a --- /dev/null +++ b/pkg/admission/plugin/resourcequota/config_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "bytes" + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" +) + +func TestLoadConfiguration(t *testing.T) { + testcases := []struct { + name string + input string + expectErr string + expectConfig *resourcequotaapi.Configuration + }{ + { + name: "empty", + input: ``, + expectErr: `'Kind' is missing`, + }, + { + name: "unknown type", + input: `{"kind":"Unknown","apiVersion":"v1"}`, + expectErr: `no kind "Unknown" is registered`, + }, + { + name: "valid v1alpha1 config", + input: ` +kind: Configuration +apiVersion: resourcequota.admission.k8s.io/v1alpha1 +limitedResources: +- apiGroup: "" + resource: persistentvolumeclaims + matchContains: + - .storageclass.storage.k8s.io/requests.storage +- apiGroup: "" + resource: pods + matchScopes: + - scopeName: PriorityClass + operator: In + values: + - cluster-services +`, + expectConfig: &resourcequotaapi.Configuration{ + LimitedResources: []resourcequotaapi.LimitedResource{ + {APIGroup: "", Resource: "persistentvolumeclaims", MatchContains: []string{".storageclass.storage.k8s.io/requests.storage"}}, + {APIGroup: "", Resource: "pods", MatchScopes: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: "PriorityClass", Operator: "In", Values: []string{"cluster-services"}}, + }, + }, + }}, + }, + { + name: "valid v1beta1 config", + input: ` +kind: Configuration +apiVersion: resourcequota.admission.k8s.io/v1beta1 +limitedResources: +- apiGroup: "" + resource: persistentvolumeclaims + matchContains: + - .storageclass.storage.k8s.io/requests.storage +- apiGroup: "" + resource: pods + matchScopes: + - scopeName: PriorityClass + operator: In + values: + - cluster-services +`, + expectConfig: &resourcequotaapi.Configuration{ + LimitedResources: []resourcequotaapi.LimitedResource{ + {APIGroup: "", Resource: "persistentvolumeclaims", MatchContains: []string{".storageclass.storage.k8s.io/requests.storage"}}, + {APIGroup: "", Resource: "pods", MatchScopes: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: "PriorityClass", Operator: "In", Values: []string{"cluster-services"}}, + }, + }, + }}, + }, + { + name: "valid v1 config", + input: ` +kind: ResourceQuotaConfiguration +apiVersion: apiserver.config.k8s.io/v1 +limitedResources: +- apiGroup: "" + resource: persistentvolumeclaims + matchContains: + - .storageclass.storage.k8s.io/requests.storage +- apiGroup: "" + resource: pods + matchScopes: + - scopeName: PriorityClass + operator: In + values: + - cluster-services +`, + expectConfig: &resourcequotaapi.Configuration{ + LimitedResources: []resourcequotaapi.LimitedResource{ + {APIGroup: "", Resource: "persistentvolumeclaims", MatchContains: []string{".storageclass.storage.k8s.io/requests.storage"}}, + {APIGroup: "", Resource: "pods", MatchScopes: []corev1.ScopedResourceSelectorRequirement{ + {ScopeName: "PriorityClass", Operator: "In", Values: []string{"cluster-services"}}, + }, + }, + }}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + config, err := LoadConfiguration(bytes.NewBuffer([]byte(tc.input))) + + if len(tc.expectErr) > 0 { + if err == nil { + t.Fatal("expected err, got none") + } + if !strings.Contains(err.Error(), tc.expectErr) { + t.Fatalf("expected err containing %q, got %v", tc.expectErr, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(config, tc.expectConfig) { + t.Fatalf("expected\n%#v\ngot\n%#v", tc.expectConfig, config) + } + }) + } +} diff --git a/pkg/admission/plugin/resourcequota/controller.go b/pkg/admission/plugin/resourcequota/controller.go new file mode 100644 index 000000000..622c7bcfd --- /dev/null +++ b/pkg/admission/plugin/resourcequota/controller.go @@ -0,0 +1,717 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "k8s.io/klog/v2" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/admission" + resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" + quota "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/apiserver/pkg/quota/v1/generic" + "k8s.io/client-go/util/workqueue" +) + +// Evaluator is used to see if quota constraints are satisfied. +type Evaluator interface { + // Evaluate takes an operation and checks to see if quota constraints are satisfied. It returns an error if they are not. + // The default implementation processes related operations in chunks when possible. + Evaluate(a admission.Attributes) error +} + +type quotaEvaluator struct { + quotaAccessor QuotaAccessor + // lockAcquisitionFunc acquires any required locks and returns a cleanup method to defer + lockAcquisitionFunc func([]corev1.ResourceQuota) func() + + ignoredResources map[schema.GroupResource]struct{} + + // registry that knows how to measure usage for objects + registry quota.Registry + + // TODO these are used together to bucket items by namespace and then batch them up for processing. + // The technique is valuable for rollup activities to avoid fanout and reduce resource contention. + // We could move this into a library if another component needed it. + // queue is indexed by namespace, so that we bundle up on a per-namespace basis + queue *workqueue.Type + workLock sync.Mutex + work map[string][]*admissionWaiter + dirtyWork map[string][]*admissionWaiter + inProgress sets.String + + // controls the run method so that we can cleanly conform to the Evaluator interface + workers int + stopCh <-chan struct{} + init sync.Once + + // lets us know what resources are limited by default + config *resourcequotaapi.Configuration +} + +type admissionWaiter struct { + attributes admission.Attributes + finished chan struct{} + result error +} + +type defaultDeny struct{} + +func (defaultDeny) Error() string { + return "DEFAULT DENY" +} + +// IsDefaultDeny returns true if the error is defaultDeny +func IsDefaultDeny(err error) bool { + if err == nil { + return false + } + + _, ok := err.(defaultDeny) + return ok +} + +func newAdmissionWaiter(a admission.Attributes) *admissionWaiter { + return &admissionWaiter{ + attributes: a, + finished: make(chan struct{}), + result: defaultDeny{}, + } +} + +// NewQuotaEvaluator configures an admission controller that can enforce quota constraints +// using the provided registry. The registry must have the capability to handle group/kinds that +// are persisted by the server this admission controller is intercepting +func NewQuotaEvaluator(quotaAccessor QuotaAccessor, ignoredResources map[schema.GroupResource]struct{}, quotaRegistry quota.Registry, lockAcquisitionFunc func([]corev1.ResourceQuota) func(), config *resourcequotaapi.Configuration, workers int, stopCh <-chan struct{}) Evaluator { + // if we get a nil config, just create an empty default. + if config == nil { + config = &resourcequotaapi.Configuration{} + } + + return "aEvaluator{ + quotaAccessor: quotaAccessor, + lockAcquisitionFunc: lockAcquisitionFunc, + + ignoredResources: ignoredResources, + registry: quotaRegistry, + + queue: workqueue.NewNamed("admission_quota_controller"), + work: map[string][]*admissionWaiter{}, + dirtyWork: map[string][]*admissionWaiter{}, + inProgress: sets.String{}, + + workers: workers, + stopCh: stopCh, + config: config, + } +} + +// Run begins watching and syncing. +func (e *quotaEvaluator) run() { + defer utilruntime.HandleCrash() + + for i := 0; i < e.workers; i++ { + go wait.Until(e.doWork, time.Second, e.stopCh) + } + <-e.stopCh + klog.Infof("Shutting down quota evaluator") + e.queue.ShutDown() +} + +func (e *quotaEvaluator) doWork() { + workFunc := func() bool { + ns, admissionAttributes, quit := e.getWork() + if quit { + return true + } + defer e.completeWork(ns) + if len(admissionAttributes) == 0 { + return false + } + e.checkAttributes(ns, admissionAttributes) + return false + } + for { + if quit := workFunc(); quit { + klog.Infof("quota evaluator worker shutdown") + return + } + } +} + +// checkAttributes iterates evaluates all the waiting admissionAttributes. It will always notify all waiters +// before returning. The default is to deny. +func (e *quotaEvaluator) checkAttributes(ns string, admissionAttributes []*admissionWaiter) { + // notify all on exit + defer func() { + for _, admissionAttribute := range admissionAttributes { + close(admissionAttribute.finished) + } + }() + + quotas, err := e.quotaAccessor.GetQuotas(ns) + if err != nil { + for _, admissionAttribute := range admissionAttributes { + admissionAttribute.result = err + } + return + } + // if limited resources are disabled, we can just return safely when there are no quotas. + limitedResourcesDisabled := len(e.config.LimitedResources) == 0 + if len(quotas) == 0 && limitedResourcesDisabled { + for _, admissionAttribute := range admissionAttributes { + admissionAttribute.result = nil + } + return + } + + if e.lockAcquisitionFunc != nil { + releaseLocks := e.lockAcquisitionFunc(quotas) + defer releaseLocks() + } + + e.checkQuotas(quotas, admissionAttributes, 3) +} + +// checkQuotas checks the admission attributes against the passed quotas. If a quota applies, it will attempt to update it +// AFTER it has checked all the admissionAttributes. The method breaks down into phase like this: +// 0. make a copy of the quotas to act as a "running" quota so we know what we need to update and can still compare against the +// originals +// 1. check each admission attribute to see if it fits within *all* the quotas. If it doesn't fit, mark the waiter as failed +// and the running quota don't change. If it did fit, check to see if any quota was changed. It there was no quota change +// mark the waiter as succeeded. If some quota did change, update the running quotas +// 2. If no running quota was changed, return now since no updates are needed. +// 3. for each quota that has changed, attempt an update. If all updates succeeded, update all unset waiters to success status and return. If the some +// updates failed on conflict errors and we have retries left, re-get the failed quota from our cache for the latest version +// and recurse into this method with the subset. It's safe for us to evaluate ONLY the subset, because the other quota +// documents for these waiters have already been evaluated. Step 1, will mark all the ones that should already have succeeded. +func (e *quotaEvaluator) checkQuotas(quotas []corev1.ResourceQuota, admissionAttributes []*admissionWaiter, remainingRetries int) { + // yet another copy to compare against originals to see if we actually have deltas + originalQuotas, err := copyQuotas(quotas) + if err != nil { + utilruntime.HandleError(err) + return + } + + atLeastOneChanged := false + for i := range admissionAttributes { + admissionAttribute := admissionAttributes[i] + newQuotas, err := e.checkRequest(quotas, admissionAttribute.attributes) + if err != nil { + admissionAttribute.result = err + continue + } + + // Don't update quota for admissionAttributes that correspond to dry-run requests + if admissionAttribute.attributes.IsDryRun() { + admissionAttribute.result = nil + continue + } + + // if the new quotas are the same as the old quotas, then this particular one doesn't issue any updates + // that means that no quota docs applied, so it can get a pass + atLeastOneChangeForThisWaiter := false + for j := range newQuotas { + if !quota.Equals(quotas[j].Status.Used, newQuotas[j].Status.Used) { + atLeastOneChanged = true + atLeastOneChangeForThisWaiter = true + break + } + } + + if !atLeastOneChangeForThisWaiter { + admissionAttribute.result = nil + } + + quotas = newQuotas + } + + // if none of the requests changed anything, there's no reason to issue an update, just fail them all now + if !atLeastOneChanged { + return + } + + // now go through and try to issue updates. Things get a little weird here: + // 1. check to see if the quota changed. If not, skip. + // 2. if the quota changed and the update passes, be happy + // 3. if the quota changed and the update fails, add the original to a retry list + var updatedFailedQuotas []corev1.ResourceQuota + var lastErr error + for i := range quotas { + newQuota := quotas[i] + + // if this quota didn't have its status changed, skip it + if quota.Equals(originalQuotas[i].Status.Used, newQuota.Status.Used) { + continue + } + + if err := e.quotaAccessor.UpdateQuotaStatus(&newQuota); err != nil { + updatedFailedQuotas = append(updatedFailedQuotas, newQuota) + lastErr = err + } + } + + if len(updatedFailedQuotas) == 0 { + // all the updates succeeded. At this point, anything with the default deny error was just waiting to + // get a successful update, so we can mark and notify + for _, admissionAttribute := range admissionAttributes { + if IsDefaultDeny(admissionAttribute.result) { + admissionAttribute.result = nil + } + } + return + } + + // at this point, errors are fatal. Update all waiters without status to failed and return + if remainingRetries <= 0 { + for _, admissionAttribute := range admissionAttributes { + if IsDefaultDeny(admissionAttribute.result) { + admissionAttribute.result = lastErr + } + } + return + } + + // this retry logic has the same bug that its possible to be checking against quota in a state that never actually exists where + // you've added a new documented, then updated an old one, your resource matches both and you're only checking one + // updates for these quota names failed. Get the current quotas in the namespace, compare by name, check to see if the + // resource versions have changed. If not, we're going to fall through an fail everything. If they all have, then we can try again + newQuotas, err := e.quotaAccessor.GetQuotas(quotas[0].Namespace) + if err != nil { + // this means that updates failed. Anything with a default deny error has failed and we need to let them know + for _, admissionAttribute := range admissionAttributes { + if IsDefaultDeny(admissionAttribute.result) { + admissionAttribute.result = lastErr + } + } + return + } + + // this logic goes through our cache to find the new version of all quotas that failed update. If something has been removed + // it is skipped on this retry. After all, you removed it. + quotasToCheck := []corev1.ResourceQuota{} + for _, newQuota := range newQuotas { + for _, oldQuota := range updatedFailedQuotas { + if newQuota.Name == oldQuota.Name { + quotasToCheck = append(quotasToCheck, newQuota) + break + } + } + } + e.checkQuotas(quotasToCheck, admissionAttributes, remainingRetries-1) +} + +func copyQuotas(in []corev1.ResourceQuota) ([]corev1.ResourceQuota, error) { + out := make([]corev1.ResourceQuota, 0, len(in)) + for _, quota := range in { + out = append(out, *quota.DeepCopy()) + } + + return out, nil +} + +// filterLimitedResourcesByGroupResource filters the input that match the specified groupResource +func filterLimitedResourcesByGroupResource(input []resourcequotaapi.LimitedResource, groupResource schema.GroupResource) []resourcequotaapi.LimitedResource { + result := []resourcequotaapi.LimitedResource{} + for i := range input { + limitedResource := input[i] + limitedGroupResource := schema.GroupResource{Group: limitedResource.APIGroup, Resource: limitedResource.Resource} + if limitedGroupResource == groupResource { + result = append(result, limitedResource) + } + } + return result +} + +// limitedByDefault determines from the specified usage and limitedResources the set of resources names +// that must be present in a covering quota. It returns empty set if it was unable to determine if +// a resource was not limited by default. +func limitedByDefault(usage corev1.ResourceList, limitedResources []resourcequotaapi.LimitedResource) []corev1.ResourceName { + result := []corev1.ResourceName{} + for _, limitedResource := range limitedResources { + for k, v := range usage { + // if a resource is consumed, we need to check if it matches on the limited resource list. + if v.Sign() == 1 { + // if we get a match, we add it to limited set + for _, matchContain := range limitedResource.MatchContains { + if strings.Contains(string(k), matchContain) { + result = append(result, k) + break + } + } + } + } + } + return result +} + +func getMatchedLimitedScopes(evaluator quota.Evaluator, inputObject runtime.Object, limitedResources []resourcequotaapi.LimitedResource) ([]corev1.ScopedResourceSelectorRequirement, error) { + scopes := []corev1.ScopedResourceSelectorRequirement{} + for _, limitedResource := range limitedResources { + matched, err := evaluator.MatchingScopes(inputObject, limitedResource.MatchScopes) + if err != nil { + klog.Errorf("Error while matching limited Scopes: %v", err) + return []corev1.ScopedResourceSelectorRequirement{}, err + } + for _, scope := range matched { + scopes = append(scopes, scope) + } + } + return scopes, nil +} + +// checkRequest verifies that the request does not exceed any quota constraint. it returns a copy of quotas not yet persisted +// that capture what the usage would be if the request succeeded. It return an error if there is insufficient quota to satisfy the request +func (e *quotaEvaluator) checkRequest(quotas []corev1.ResourceQuota, a admission.Attributes) ([]corev1.ResourceQuota, error) { + evaluator := e.registry.Get(a.GetResource().GroupResource()) + if evaluator == nil { + return quotas, nil + } + return CheckRequest(quotas, a, evaluator, e.config.LimitedResources) +} + +// CheckRequest is a static version of quotaEvaluator.checkRequest, possible to be called from outside. +func CheckRequest(quotas []corev1.ResourceQuota, a admission.Attributes, evaluator quota.Evaluator, + limited []resourcequotaapi.LimitedResource) ([]corev1.ResourceQuota, error) { + if !evaluator.Handles(a) { + return quotas, nil + } + + // if we have limited resources enabled for this resource, always calculate usage + inputObject := a.GetObject() + + // Check if object matches AdmissionConfiguration matchScopes + limitedScopes, err := getMatchedLimitedScopes(evaluator, inputObject, limited) + if err != nil { + return quotas, nil + } + + // determine the set of resource names that must exist in a covering quota + limitedResourceNames := []corev1.ResourceName{} + limitedResources := filterLimitedResourcesByGroupResource(limited, a.GetResource().GroupResource()) + if len(limitedResources) > 0 { + deltaUsage, err := evaluator.Usage(inputObject) + if err != nil { + return quotas, err + } + limitedResourceNames = limitedByDefault(deltaUsage, limitedResources) + } + limitedResourceNamesSet := quota.ToSet(limitedResourceNames) + + // find the set of quotas that are pertinent to this request + // reject if we match the quota, but usage is not calculated yet + // reject if the input object does not satisfy quota constraints + // if there are no pertinent quotas, we can just return + interestingQuotaIndexes := []int{} + // track the cumulative set of resources that were required across all quotas + // this is needed to know if we have satisfied any constraints where consumption + // was limited by default. + restrictedResourcesSet := sets.String{} + restrictedScopes := []corev1.ScopedResourceSelectorRequirement{} + for i := range quotas { + resourceQuota := quotas[i] + scopeSelectors := getScopeSelectorsFromQuota(resourceQuota) + localRestrictedScopes, err := evaluator.MatchingScopes(inputObject, scopeSelectors) + if err != nil { + return nil, fmt.Errorf("error matching scopes of quota %s, err: %v", resourceQuota.Name, err) + } + for _, scope := range localRestrictedScopes { + restrictedScopes = append(restrictedScopes, scope) + } + + match, err := evaluator.Matches(&resourceQuota, inputObject) + if err != nil { + klog.Errorf("Error occurred while matching resource quota, %v, against input object. Err: %v", resourceQuota, err) + return quotas, err + } + if !match { + continue + } + + hardResources := quota.ResourceNames(resourceQuota.Status.Hard) + restrictedResources := evaluator.MatchingResources(hardResources) + if err := evaluator.Constraints(restrictedResources, inputObject); err != nil { + return nil, admission.NewForbidden(a, fmt.Errorf("failed quota: %s: %v", resourceQuota.Name, err)) + } + if !hasUsageStats(&resourceQuota, restrictedResources) { + return nil, admission.NewForbidden(a, fmt.Errorf("status unknown for quota: %s, resources: %s", resourceQuota.Name, prettyPrintResourceNames(restrictedResources))) + } + interestingQuotaIndexes = append(interestingQuotaIndexes, i) + localRestrictedResourcesSet := quota.ToSet(restrictedResources) + restrictedResourcesSet.Insert(localRestrictedResourcesSet.List()...) + } + + // Usage of some resources cannot be counted in isolation. For example, when + // the resource represents a number of unique references to external + // resource. In such a case an evaluator needs to process other objects in + // the same namespace which needs to be known. + namespace := a.GetNamespace() + if accessor, err := meta.Accessor(inputObject); namespace != "" && err == nil { + if accessor.GetNamespace() == "" { + accessor.SetNamespace(namespace) + } + } + // there is at least one quota that definitely matches our object + // as a result, we need to measure the usage of this object for quota + // on updates, we need to subtract the previous measured usage + // if usage shows no change, just return since it has no impact on quota + deltaUsage, err := evaluator.Usage(inputObject) + if err != nil { + return quotas, err + } + + // ensure that usage for input object is never negative (this would mean a resource made a negative resource requirement) + if negativeUsage := quota.IsNegative(deltaUsage); len(negativeUsage) > 0 { + return nil, admission.NewForbidden(a, fmt.Errorf("quota usage is negative for resource(s): %s", prettyPrintResourceNames(negativeUsage))) + } + + if admission.Update == a.GetOperation() { + prevItem := a.GetOldObject() + if prevItem == nil { + return nil, admission.NewForbidden(a, fmt.Errorf("unable to get previous usage since prior version of object was not found")) + } + + // if we can definitively determine that this is not a case of "create on update", + // then charge based on the delta. Otherwise, bill the maximum + metadata, err := meta.Accessor(prevItem) + if err == nil && len(metadata.GetResourceVersion()) > 0 { + prevUsage, innerErr := evaluator.Usage(prevItem) + if innerErr != nil { + return quotas, innerErr + } + deltaUsage = quota.SubtractWithNonNegativeResult(deltaUsage, prevUsage) + } + } + + if quota.IsZero(deltaUsage) { + return quotas, nil + } + + // verify that for every resource that had limited by default consumption + // enabled that there was a corresponding quota that covered its use. + // if not, we reject the request. + hasNoCoveringQuota := limitedResourceNamesSet.Difference(restrictedResourcesSet) + if len(hasNoCoveringQuota) > 0 { + return quotas, admission.NewForbidden(a, fmt.Errorf("insufficient quota to consume: %v", strings.Join(hasNoCoveringQuota.List(), ","))) + } + + // verify that for every scope that had limited access enabled + // that there was a corresponding quota that covered it. + // if not, we reject the request. + scopesHasNoCoveringQuota, err := evaluator.UncoveredQuotaScopes(limitedScopes, restrictedScopes) + if err != nil { + return quotas, err + } + if len(scopesHasNoCoveringQuota) > 0 { + return quotas, fmt.Errorf("insufficient quota to match these scopes: %v", scopesHasNoCoveringQuota) + } + + if len(interestingQuotaIndexes) == 0 { + return quotas, nil + } + + outQuotas, err := copyQuotas(quotas) + if err != nil { + return nil, err + } + + for _, index := range interestingQuotaIndexes { + resourceQuota := outQuotas[index] + + hardResources := quota.ResourceNames(resourceQuota.Status.Hard) + requestedUsage := quota.Mask(deltaUsage, hardResources) + newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage) + maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage)) + + if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed { + failedRequestedUsage := quota.Mask(requestedUsage, exceeded) + failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded) + failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded) + return nil, admission.NewForbidden(a, + fmt.Errorf("exceeded quota: %s, requested: %s, used: %s, limited: %s", + resourceQuota.Name, + prettyPrint(failedRequestedUsage), + prettyPrint(failedUsed), + prettyPrint(failedHard))) + } + + // update to the new usage number + outQuotas[index].Status.Used = newUsage + } + + return outQuotas, nil +} + +func getScopeSelectorsFromQuota(quota corev1.ResourceQuota) []corev1.ScopedResourceSelectorRequirement { + selectors := []corev1.ScopedResourceSelectorRequirement{} + for _, scope := range quota.Spec.Scopes { + selectors = append(selectors, corev1.ScopedResourceSelectorRequirement{ + ScopeName: scope, + Operator: corev1.ScopeSelectorOpExists}) + } + if quota.Spec.ScopeSelector != nil { + for _, scopeSelector := range quota.Spec.ScopeSelector.MatchExpressions { + selectors = append(selectors, scopeSelector) + } + } + return selectors +} + +func (e *quotaEvaluator) Evaluate(a admission.Attributes) error { + e.init.Do(func() { + go e.run() + }) + + // is this resource ignored? + gvr := a.GetResource() + gr := gvr.GroupResource() + if _, ok := e.ignoredResources[gr]; ok { + return nil + } + + // if we do not know how to evaluate use for this resource, create an evaluator + evaluator := e.registry.Get(gr) + if evaluator == nil { + // create an object count evaluator if no evaluator previously registered + // note, we do not need aggregate usage here, so we pass a nil informer func + evaluator = generic.NewObjectCountEvaluator(gr, nil, "") + e.registry.Add(evaluator) + klog.Infof("quota admission added evaluator for: %s", gr) + } + // for this kind, check if the operation could mutate any quota resources + // if no resources tracked by quota are impacted, then just return + if !evaluator.Handles(a) { + return nil + } + waiter := newAdmissionWaiter(a) + + e.addWork(waiter) + + // wait for completion or timeout + select { + case <-waiter.finished: + case <-time.After(10 * time.Second): + return apierrors.NewInternalError(fmt.Errorf("resource quota evaluation timed out")) + } + + return waiter.result +} + +func (e *quotaEvaluator) addWork(a *admissionWaiter) { + e.workLock.Lock() + defer e.workLock.Unlock() + + ns := a.attributes.GetNamespace() + // this Add can trigger a Get BEFORE the work is added to a list, but this is ok because the getWork routine + // waits the worklock before retrieving the work to do, so the writes in this method will be observed + e.queue.Add(ns) + + if e.inProgress.Has(ns) { + e.dirtyWork[ns] = append(e.dirtyWork[ns], a) + return + } + + e.work[ns] = append(e.work[ns], a) +} + +func (e *quotaEvaluator) completeWork(ns string) { + e.workLock.Lock() + defer e.workLock.Unlock() + + e.queue.Done(ns) + e.work[ns] = e.dirtyWork[ns] + delete(e.dirtyWork, ns) + e.inProgress.Delete(ns) +} + +// getWork returns a namespace, a list of work items in that +// namespace, and a shutdown boolean. If not shutdown then the return +// must eventually be followed by a call on completeWork for the +// returned namespace (regardless of whether the work item list is +// empty). +func (e *quotaEvaluator) getWork() (string, []*admissionWaiter, bool) { + uncastNS, shutdown := e.queue.Get() + if shutdown { + return "", []*admissionWaiter{}, shutdown + } + ns := uncastNS.(string) + + e.workLock.Lock() + defer e.workLock.Unlock() + // at this point, we know we have a coherent view of e.work. It is entirely possible + // that our workqueue has another item requeued to it, but we'll pick it up early. This ok + // because the next time will go into our dirty list + + work := e.work[ns] + delete(e.work, ns) + delete(e.dirtyWork, ns) + e.inProgress.Insert(ns) + return ns, work, false +} + +// prettyPrint formats a resource list for usage in errors +// it outputs resources sorted in increasing order +func prettyPrint(item corev1.ResourceList) string { + parts := []string{} + keys := []string{} + for key := range item { + keys = append(keys, string(key)) + } + sort.Strings(keys) + for _, key := range keys { + value := item[corev1.ResourceName(key)] + constraint := key + "=" + value.String() + parts = append(parts, constraint) + } + return strings.Join(parts, ",") +} + +func prettyPrintResourceNames(a []corev1.ResourceName) string { + values := []string{} + for _, value := range a { + values = append(values, string(value)) + } + sort.Strings(values) + return strings.Join(values, ",") +} + +// hasUsageStats returns true if for each hard constraint in interestingResources there is a value for its current usage +func hasUsageStats(resourceQuota *corev1.ResourceQuota, interestingResources []corev1.ResourceName) bool { + interestingSet := quota.ToSet(interestingResources) + for resourceName := range resourceQuota.Status.Hard { + if !interestingSet.Has(string(resourceName)) { + continue + } + if _, found := resourceQuota.Status.Used[resourceName]; !found { + return false + } + } + return true +} diff --git a/pkg/admission/plugin/resourcequota/doc.go b/pkg/admission/plugin/resourcequota/doc.go new file mode 100644 index 000000000..4436e3303 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resourcequota enforces all incoming requests against any applied quota +// in the namespace context of the request +package resourcequota // import "k8s.io/apiserver/pkg/admission/plugin/resourcequota" diff --git a/pkg/admission/plugin/resourcequota/resource_access.go b/pkg/admission/plugin/resourcequota/resource_access.go new file mode 100644 index 000000000..9955402b9 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/resource_access.go @@ -0,0 +1,155 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "context" + "fmt" + "time" + + "github.com/hashicorp/golang-lru" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apiserver/pkg/storage/etcd3" + "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" +) + +// QuotaAccessor abstracts the get/set logic from the rest of the Evaluator. This could be a test stub, a straight passthrough, +// or most commonly a series of deconflicting caches. +type QuotaAccessor interface { + // UpdateQuotaStatus is called to persist final status. This method should write to persistent storage. + // An error indicates that write didn't complete successfully. + UpdateQuotaStatus(newQuota *corev1.ResourceQuota) error + + // GetQuotas gets all possible quotas for a given namespace + GetQuotas(namespace string) ([]corev1.ResourceQuota, error) +} + +type quotaAccessor struct { + client kubernetes.Interface + + // lister can list/get quota objects from a shared informer's cache + lister corev1listers.ResourceQuotaLister + + // liveLookups holds the last few live lookups we've done to help ammortize cost on repeated lookup failures. + // This lets us handle the case of latent caches, by looking up actual results for a namespace on cache miss/no results. + // We track the lookup result here so that for repeated requests, we don't look it up very often. + liveLookupCache *lru.Cache + liveTTL time.Duration + // updatedQuotas holds a cache of quotas that we've updated. This is used to pull the "really latest" during back to + // back quota evaluations that touch the same quota doc. This only works because we can compare etcd resourceVersions + // for the same resource as integers. Before this change: 22 updates with 12 conflicts. after this change: 15 updates with 0 conflicts + updatedQuotas *lru.Cache +} + +// newQuotaAccessor creates an object that conforms to the QuotaAccessor interface to be used to retrieve quota objects. +func newQuotaAccessor() (*quotaAccessor, error) { + liveLookupCache, err := lru.New(100) + if err != nil { + return nil, err + } + updatedCache, err := lru.New(100) + if err != nil { + return nil, err + } + + // client and lister will be set when SetInternalKubeClientSet and SetInternalKubeInformerFactory are invoked + return "aAccessor{ + liveLookupCache: liveLookupCache, + liveTTL: time.Duration(30 * time.Second), + updatedQuotas: updatedCache, + }, nil +} + +func (e *quotaAccessor) UpdateQuotaStatus(newQuota *corev1.ResourceQuota) error { + updatedQuota, err := e.client.CoreV1().ResourceQuotas(newQuota.Namespace).UpdateStatus(context.TODO(), newQuota, metav1.UpdateOptions{}) + if err != nil { + return err + } + + key := newQuota.Namespace + "/" + newQuota.Name + e.updatedQuotas.Add(key, updatedQuota) + return nil +} + +var etcdVersioner = etcd3.APIObjectVersioner{} + +// checkCache compares the passed quota against the value in the look-aside cache and returns the newer +// if the cache is out of date, it deletes the stale entry. This only works because of etcd resourceVersions +// being monotonically increasing integers +func (e *quotaAccessor) checkCache(quota *corev1.ResourceQuota) *corev1.ResourceQuota { + key := quota.Namespace + "/" + quota.Name + uncastCachedQuota, ok := e.updatedQuotas.Get(key) + if !ok { + return quota + } + cachedQuota := uncastCachedQuota.(*corev1.ResourceQuota) + + if etcdVersioner.CompareResourceVersion(quota, cachedQuota) >= 0 { + e.updatedQuotas.Remove(key) + return quota + } + return cachedQuota +} + +func (e *quotaAccessor) GetQuotas(namespace string) ([]corev1.ResourceQuota, error) { + // determine if there are any quotas in this namespace + // if there are no quotas, we don't need to do anything + items, err := e.lister.ResourceQuotas(namespace).List(labels.Everything()) + if err != nil { + return nil, fmt.Errorf("error resolving quota: %v", err) + } + + // if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it. + if len(items) == 0 { + lruItemObj, ok := e.liveLookupCache.Get(namespace) + if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) { + // TODO: If there are multiple operations at the same time and cache has just expired, + // this may cause multiple List operations being issued at the same time. + // If there is already in-flight List() for a given namespace, we should wait until + // it is finished and cache is updated instead of doing the same, also to avoid + // throttling - see #22422 for details. + liveList, err := e.client.CoreV1().ResourceQuotas(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + newEntry := liveLookupEntry{expiry: time.Now().Add(e.liveTTL)} + for i := range liveList.Items { + newEntry.items = append(newEntry.items, &liveList.Items[i]) + } + e.liveLookupCache.Add(namespace, newEntry) + lruItemObj = newEntry + } + lruEntry := lruItemObj.(liveLookupEntry) + for i := range lruEntry.items { + items = append(items, lruEntry.items[i]) + } + } + + resourceQuotas := []corev1.ResourceQuota{} + for i := range items { + quota := items[i] + quota = e.checkCache(quota) + // always make a copy. We're going to muck around with this and we should never mutate the originals + resourceQuotas = append(resourceQuotas, *quota) + } + + return resourceQuotas, nil +} diff --git a/pkg/admission/plugin/resourcequota/resource_access_test.go b/pkg/admission/plugin/resourcequota/resource_access_test.go new file mode 100644 index 000000000..48c4583a7 --- /dev/null +++ b/pkg/admission/plugin/resourcequota/resource_access_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcequota + +import ( + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + lru "github.com/hashicorp/golang-lru" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +func TestLRUCacheLookup(t *testing.T) { + namespace := "foo" + resourceQuota := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: namespace, + }, + } + + testcases := []struct { + description string + cacheInput []*corev1.ResourceQuota + clientInput []runtime.Object + ttl time.Duration + namespace string + expectedQuota *corev1.ResourceQuota + }{ + { + description: "object is found via cache", + cacheInput: []*corev1.ResourceQuota{resourceQuota}, + ttl: 30 * time.Second, + namespace: namespace, + expectedQuota: resourceQuota, + }, + { + description: "object is outdated and not found with client", + cacheInput: []*corev1.ResourceQuota{resourceQuota}, + ttl: -30 * time.Second, + namespace: namespace, + expectedQuota: nil, + }, + { + description: "object is outdated but is found with client", + cacheInput: []*corev1.ResourceQuota{resourceQuota}, + clientInput: []runtime.Object{resourceQuota}, + ttl: -30 * time.Second, + namespace: namespace, + expectedQuota: resourceQuota, + }, + { + description: "object does not exist in cache and is not found with client", + cacheInput: []*corev1.ResourceQuota{resourceQuota}, + ttl: 30 * time.Second, + expectedQuota: nil, + }, + { + description: "object does not exist in cache and is found with client", + cacheInput: []*corev1.ResourceQuota{}, + clientInput: []runtime.Object{resourceQuota}, + namespace: namespace, + expectedQuota: resourceQuota, + }, + } + + for _, tc := range testcases { + t.Run(tc.description, func(t *testing.T) { + liveLookupCache, err := lru.New(1) + if err != nil { + t.Fatal(err) + } + kubeClient := fake.NewSimpleClientset(tc.clientInput...) + informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + + accessor, _ := newQuotaAccessor() + accessor.client = kubeClient + accessor.lister = informerFactory.Core().V1().ResourceQuotas().Lister() + accessor.liveLookupCache = liveLookupCache + + for _, q := range tc.cacheInput { + quota := q + liveLookupCache.Add(quota.Namespace, liveLookupEntry{expiry: time.Now().Add(tc.ttl), items: []*corev1.ResourceQuota{quota}}) + } + + quotas, err := accessor.GetQuotas(tc.namespace) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if tc.expectedQuota != nil { + if count := len(quotas); count != 1 { + t.Fatalf("Expected 1 object but got %d", count) + } + + if !reflect.DeepEqual(quotas[0], *tc.expectedQuota) { + t.Errorf("Retrieved object does not match") + } + return + } + + if count := len(quotas); count > 0 { + t.Errorf("Expected 0 objects but got %d", count) + } + }) + } + +} diff --git a/pkg/quota/v1/OWNERS b/pkg/quota/v1/OWNERS new file mode 100644 index 000000000..d812b5d3e --- /dev/null +++ b/pkg/quota/v1/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- deads2k +- derekwaynecarr +- vishh +reviewers: +- deads2k +- derekwaynecarr +- smarterclayton +- vishh +labels: +- sig/api-machinery diff --git a/pkg/quota/v1/generic/OWNERS b/pkg/quota/v1/generic/OWNERS new file mode 100644 index 000000000..4d4f90e4d --- /dev/null +++ b/pkg/quota/v1/generic/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: +- smarterclayton +- derekwaynecarr diff --git a/pkg/quota/v1/generic/configuration.go b/pkg/quota/v1/generic/configuration.go new file mode 100644 index 000000000..966c5c7c6 --- /dev/null +++ b/pkg/quota/v1/generic/configuration.go @@ -0,0 +1,44 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + quota "k8s.io/apiserver/pkg/quota/v1" +) + +// implements a basic configuration +type simpleConfiguration struct { + evaluators []quota.Evaluator + ignoredResources map[schema.GroupResource]struct{} +} + +// NewConfiguration creates a quota configuration +func NewConfiguration(evaluators []quota.Evaluator, ignoredResources map[schema.GroupResource]struct{}) quota.Configuration { + return &simpleConfiguration{ + evaluators: evaluators, + ignoredResources: ignoredResources, + } +} + +func (c *simpleConfiguration) IgnoredResources() map[schema.GroupResource]struct{} { + return c.ignoredResources +} + +func (c *simpleConfiguration) Evaluators() []quota.Evaluator { + return c.evaluators +} diff --git a/pkg/quota/v1/generic/evaluator.go b/pkg/quota/v1/generic/evaluator.go new file mode 100644 index 000000000..7ba48c942 --- /dev/null +++ b/pkg/quota/v1/generic/evaluator.go @@ -0,0 +1,319 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "fmt" + "sync/atomic" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + quota "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" +) + +// InformerForResourceFunc knows how to provision an informer +type InformerForResourceFunc func(schema.GroupVersionResource) (informers.GenericInformer, error) + +// ListerFuncForResourceFunc knows how to provision a lister from an informer func. +// The lister returns errors until the informer has synced. +func ListerFuncForResourceFunc(f InformerForResourceFunc) quota.ListerForResourceFunc { + return func(gvr schema.GroupVersionResource) (cache.GenericLister, error) { + informer, err := f(gvr) + if err != nil { + return nil, err + } + return &protectedLister{ + hasSynced: cachedHasSynced(informer.Informer().HasSynced), + notReadyErr: fmt.Errorf("%v not yet synced", gvr), + delegate: informer.Lister(), + }, nil + } +} + +// cachedHasSynced returns a function that calls hasSynced() until it returns true once, then returns true +func cachedHasSynced(hasSynced func() bool) func() bool { + cache := &atomic.Value{} + cache.Store(false) + return func() bool { + if cache.Load().(bool) { + // short-circuit if already synced + return true + } + if hasSynced() { + // remember we synced + cache.Store(true) + return true + } + return false + } +} + +// protectedLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate +type protectedLister struct { + hasSynced func() bool + notReadyErr error + delegate cache.GenericLister +} + +func (p *protectedLister) List(selector labels.Selector) (ret []runtime.Object, err error) { + if !p.hasSynced() { + return nil, p.notReadyErr + } + return p.delegate.List(selector) +} +func (p *protectedLister) Get(name string) (runtime.Object, error) { + if !p.hasSynced() { + return nil, p.notReadyErr + } + return p.delegate.Get(name) +} +func (p *protectedLister) ByNamespace(namespace string) cache.GenericNamespaceLister { + return &protectedNamespaceLister{p.hasSynced, p.notReadyErr, p.delegate.ByNamespace(namespace)} +} + +// protectedNamespaceLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate +type protectedNamespaceLister struct { + hasSynced func() bool + notReadyErr error + delegate cache.GenericNamespaceLister +} + +func (p *protectedNamespaceLister) List(selector labels.Selector) (ret []runtime.Object, err error) { + if !p.hasSynced() { + return nil, p.notReadyErr + } + return p.delegate.List(selector) +} +func (p *protectedNamespaceLister) Get(name string) (runtime.Object, error) { + if !p.hasSynced() { + return nil, p.notReadyErr + } + return p.delegate.Get(name) +} + +// ListResourceUsingListerFunc returns a listing function based on the shared informer factory for the specified resource. +func ListResourceUsingListerFunc(l quota.ListerForResourceFunc, resource schema.GroupVersionResource) ListFuncByNamespace { + return func(namespace string) ([]runtime.Object, error) { + lister, err := l(resource) + if err != nil { + return nil, err + } + return lister.ByNamespace(namespace).List(labels.Everything()) + } +} + +// ObjectCountQuotaResourceNameFor returns the object count quota name for specified groupResource +func ObjectCountQuotaResourceNameFor(groupResource schema.GroupResource) corev1.ResourceName { + if len(groupResource.Group) == 0 { + return corev1.ResourceName("count/" + groupResource.Resource) + } + return corev1.ResourceName("count/" + groupResource.Resource + "." + groupResource.Group) +} + +// ListFuncByNamespace knows how to list resources in a namespace +type ListFuncByNamespace func(namespace string) ([]runtime.Object, error) + +// MatchesScopeFunc knows how to evaluate if an object matches a scope +type MatchesScopeFunc func(scope corev1.ScopedResourceSelectorRequirement, object runtime.Object) (bool, error) + +// UsageFunc knows how to measure usage associated with an object +type UsageFunc func(object runtime.Object) (corev1.ResourceList, error) + +// MatchingResourceNamesFunc is a function that returns the list of resources matched +type MatchingResourceNamesFunc func(input []corev1.ResourceName) []corev1.ResourceName + +// MatchesNoScopeFunc returns false on all match checks +func MatchesNoScopeFunc(scope corev1.ScopedResourceSelectorRequirement, object runtime.Object) (bool, error) { + return false, nil +} + +// Matches returns true if the quota matches the specified item. +func Matches( + resourceQuota *corev1.ResourceQuota, item runtime.Object, + matchFunc MatchingResourceNamesFunc, scopeFunc MatchesScopeFunc) (bool, error) { + if resourceQuota == nil { + return false, fmt.Errorf("expected non-nil quota") + } + // verify the quota matches on at least one resource + matchResource := len(matchFunc(quota.ResourceNames(resourceQuota.Status.Hard))) > 0 + // by default, no scopes matches all + matchScope := true + for _, scope := range getScopeSelectorsFromQuota(resourceQuota) { + innerMatch, err := scopeFunc(scope, item) + if err != nil { + return false, err + } + matchScope = matchScope && innerMatch + } + return matchResource && matchScope, nil +} + +func getScopeSelectorsFromQuota(quota *corev1.ResourceQuota) []corev1.ScopedResourceSelectorRequirement { + selectors := []corev1.ScopedResourceSelectorRequirement{} + for _, scope := range quota.Spec.Scopes { + selectors = append(selectors, corev1.ScopedResourceSelectorRequirement{ + ScopeName: scope, + Operator: corev1.ScopeSelectorOpExists}) + } + if quota.Spec.ScopeSelector != nil { + selectors = append(selectors, quota.Spec.ScopeSelector.MatchExpressions...) + } + return selectors +} + +// CalculateUsageStats is a utility function that knows how to calculate aggregate usage. +func CalculateUsageStats(options quota.UsageStatsOptions, + listFunc ListFuncByNamespace, + scopeFunc MatchesScopeFunc, + usageFunc UsageFunc) (quota.UsageStats, error) { + // default each tracked resource to zero + result := quota.UsageStats{Used: corev1.ResourceList{}} + for _, resourceName := range options.Resources { + result.Used[resourceName] = resource.Quantity{Format: resource.DecimalSI} + } + items, err := listFunc(options.Namespace) + if err != nil { + return result, fmt.Errorf("failed to list content: %v", err) + } + for _, item := range items { + // need to verify that the item matches the set of scopes + matchesScopes := true + for _, scope := range options.Scopes { + innerMatch, err := scopeFunc(corev1.ScopedResourceSelectorRequirement{ScopeName: scope}, item) + if err != nil { + return result, nil + } + if !innerMatch { + matchesScopes = false + } + } + if options.ScopeSelector != nil { + for _, selector := range options.ScopeSelector.MatchExpressions { + innerMatch, err := scopeFunc(selector, item) + if err != nil { + return result, nil + } + matchesScopes = matchesScopes && innerMatch + } + } + // only count usage if there was a match + if matchesScopes { + usage, err := usageFunc(item) + if err != nil { + return result, err + } + result.Used = quota.Add(result.Used, usage) + } + } + return result, nil +} + +// objectCountEvaluator provides an implementation for quota.Evaluator +// that associates usage of the specified resource based on the number of items +// returned by the specified listing function. +type objectCountEvaluator struct { + // GroupResource that this evaluator tracks. + // It is used to construct a generic object count quota name + groupResource schema.GroupResource + // A function that knows how to list resources by namespace. + // TODO move to dynamic client in future + listFuncByNamespace ListFuncByNamespace + // Names associated with this resource in the quota for generic counting. + resourceNames []corev1.ResourceName +} + +// Constraints returns an error if the configured resource name is not in the required set. +func (o *objectCountEvaluator) Constraints(required []corev1.ResourceName, item runtime.Object) error { + // no-op for object counting + return nil +} + +// Handles returns true if the object count evaluator needs to track this attributes. +func (o *objectCountEvaluator) Handles(a admission.Attributes) bool { + operation := a.GetOperation() + return operation == admission.Create +} + +// Matches returns true if the evaluator matches the specified quota with the provided input item +func (o *objectCountEvaluator) Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error) { + return Matches(resourceQuota, item, o.MatchingResources, MatchesNoScopeFunc) +} + +// MatchingResources takes the input specified list of resources and returns the set of resources it matches. +func (o *objectCountEvaluator) MatchingResources(input []corev1.ResourceName) []corev1.ResourceName { + return quota.Intersection(input, o.resourceNames) +} + +// MatchingScopes takes the input specified list of scopes and input object. Returns the set of scopes resource matches. +func (o *objectCountEvaluator) MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + return []corev1.ScopedResourceSelectorRequirement{}, nil +} + +// UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes. +// It returns the scopes which are in limited scopes but dont have a corresponding covering quota scope +func (o *objectCountEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) { + return []corev1.ScopedResourceSelectorRequirement{}, nil +} + +// Usage returns the resource usage for the specified object +func (o *objectCountEvaluator) Usage(object runtime.Object) (corev1.ResourceList, error) { + quantity := resource.NewQuantity(1, resource.DecimalSI) + resourceList := corev1.ResourceList{} + for _, resourceName := range o.resourceNames { + resourceList[resourceName] = *quantity + } + return resourceList, nil +} + +// GroupResource tracked by this evaluator +func (o *objectCountEvaluator) GroupResource() schema.GroupResource { + return o.groupResource +} + +// UsageStats calculates aggregate usage for the object. +func (o *objectCountEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) { + return CalculateUsageStats(options, o.listFuncByNamespace, MatchesNoScopeFunc, o.Usage) +} + +// Verify implementation of interface at compile time. +var _ quota.Evaluator = &objectCountEvaluator{} + +// NewObjectCountEvaluator returns an evaluator that can perform generic +// object quota counting. It allows an optional alias for backwards compatibility +// purposes for the legacy object counting names in quota. Unless its supporting +// backward compatibility, alias should not be used. +func NewObjectCountEvaluator( + groupResource schema.GroupResource, listFuncByNamespace ListFuncByNamespace, + alias corev1.ResourceName) quota.Evaluator { + + resourceNames := []corev1.ResourceName{ObjectCountQuotaResourceNameFor(groupResource)} + if len(alias) > 0 { + resourceNames = append(resourceNames, alias) + } + + return &objectCountEvaluator{ + groupResource: groupResource, + listFuncByNamespace: listFuncByNamespace, + resourceNames: resourceNames, + } +} diff --git a/pkg/quota/v1/generic/evaluator_test.go b/pkg/quota/v1/generic/evaluator_test.go new file mode 100644 index 000000000..ce97b1bdf --- /dev/null +++ b/pkg/quota/v1/generic/evaluator_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "errors" + "testing" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" +) + +func TestCachedHasSynced(t *testing.T) { + + called := 0 + result := false + cachedFunc := cachedHasSynced(func() bool { + called++ + return result + }) + + if cachedFunc() { + t.Fatal("expected false") + } + if called != 1 { + t.Fatalf("expected called=1, got %d", called) + } + + if cachedFunc() { + t.Fatal("expected false") + } + if called != 2 { + t.Fatalf("expected called=2, got %d", called) + } + + result = true + if !cachedFunc() { + t.Fatal("expected true") + } + if called != 3 { + t.Fatalf("expected called=3, got %d", called) + } + + if !cachedFunc() { + t.Fatal("expected true") + } + if called != 3 { + // no more calls once we return true + t.Fatalf("expected called=3, got %d", called) + } +} + +func TestProtectedLister(t *testing.T) { + + hasSynced := false + notReadyErr := errors.New("not ready") + fake := &fakeLister{} + l := &protectedLister{ + hasSynced: func() bool { return hasSynced }, + notReadyErr: notReadyErr, + delegate: fake, + } + if _, err := l.List(nil); err != notReadyErr { + t.Fatalf("expected %v, got %v", notReadyErr, err) + } + if _, err := l.Get(""); err != notReadyErr { + t.Fatalf("expected %v, got %v", notReadyErr, err) + } + if fake.called != 0 { + t.Fatalf("expected called=0, got %d", fake.called) + } + fake.called = 0 + + hasSynced = true + + if _, err := l.List(nil); err != errFakeLister { + t.Fatalf("expected %v, got %v", errFakeLister, err) + } + if _, err := l.Get(""); err != errFakeLister { + t.Fatalf("expected %v, got %v", errFakeLister, err) + } + if fake.called != 2 { + t.Fatalf("expected called=2, got %d", fake.called) + } + fake.called = 0 + + hasSynced = false + + if _, err := l.List(nil); err != notReadyErr { + t.Fatalf("expected %v, got %v", notReadyErr, err) + } + if _, err := l.Get(""); err != notReadyErr { + t.Fatalf("expected %v, got %v", notReadyErr, err) + } + if fake.called != 0 { + t.Fatalf("expected called=2, got %d", fake.called) + } +} + +var errFakeLister = errors.New("errFakeLister") + +type fakeLister struct { + called int +} + +func (f *fakeLister) List(selector labels.Selector) (ret []runtime.Object, err error) { + f.called++ + return nil, errFakeLister +} +func (f *fakeLister) Get(name string) (runtime.Object, error) { + f.called++ + return nil, errFakeLister +} +func (f *fakeLister) ByNamespace(namespace string) cache.GenericNamespaceLister { + panic("not implemented") +} diff --git a/pkg/quota/v1/generic/registry.go b/pkg/quota/v1/generic/registry.go new file mode 100644 index 000000000..0c1023606 --- /dev/null +++ b/pkg/quota/v1/generic/registry.go @@ -0,0 +1,81 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "sync" + + "k8s.io/apimachinery/pkg/runtime/schema" + quota "k8s.io/apiserver/pkg/quota/v1" +) + +// implements a basic registry +type simpleRegistry struct { + lock sync.RWMutex + // evaluators tracked by the registry + evaluators map[schema.GroupResource]quota.Evaluator +} + +// NewRegistry creates a simple registry with initial list of evaluators +func NewRegistry(evaluators []quota.Evaluator) quota.Registry { + return &simpleRegistry{ + evaluators: evaluatorsByGroupResource(evaluators), + } +} + +func (r *simpleRegistry) Add(e quota.Evaluator) { + r.lock.Lock() + defer r.lock.Unlock() + r.evaluators[e.GroupResource()] = e +} + +func (r *simpleRegistry) Remove(e quota.Evaluator) { + r.lock.Lock() + defer r.lock.Unlock() + delete(r.evaluators, e.GroupResource()) +} + +func (r *simpleRegistry) Get(gr schema.GroupResource) quota.Evaluator { + r.lock.RLock() + defer r.lock.RUnlock() + return r.evaluators[gr] +} + +func (r *simpleRegistry) List() []quota.Evaluator { + r.lock.RLock() + defer r.lock.RUnlock() + + return evaluatorsList(r.evaluators) +} + +// evaluatorsByGroupResource converts a list of evaluators to a map by group resource. +func evaluatorsByGroupResource(items []quota.Evaluator) map[schema.GroupResource]quota.Evaluator { + result := map[schema.GroupResource]quota.Evaluator{} + for _, item := range items { + result[item.GroupResource()] = item + } + return result +} + +// evaluatorsList converts a map of evaluators to list +func evaluatorsList(input map[schema.GroupResource]quota.Evaluator) []quota.Evaluator { + var result []quota.Evaluator + for _, item := range input { + result = append(result, item) + } + return result +} diff --git a/pkg/quota/v1/interfaces.go b/pkg/quota/v1/interfaces.go new file mode 100644 index 000000000..15f8b7613 --- /dev/null +++ b/pkg/quota/v1/interfaces.go @@ -0,0 +1,88 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/client-go/tools/cache" +) + +// UsageStatsOptions is an options structs that describes how stats should be calculated +type UsageStatsOptions struct { + // Namespace where stats should be calculate + Namespace string + // Scopes that must match counted objects + Scopes []corev1.ResourceQuotaScope + // Resources are the set of resources to include in the measurement + Resources []corev1.ResourceName + ScopeSelector *corev1.ScopeSelector +} + +// UsageStats is result of measuring observed resource use in the system +type UsageStats struct { + // Used maps resource to quantity used + Used corev1.ResourceList +} + +// Evaluator knows how to evaluate quota usage for a particular group resource +type Evaluator interface { + // Constraints ensures that each required resource is present on item + Constraints(required []corev1.ResourceName, item runtime.Object) error + // GroupResource returns the groupResource that this object knows how to evaluate + GroupResource() schema.GroupResource + // Handles determines if quota could be impacted by the specified attribute. + // If true, admission control must perform quota processing for the operation, otherwise it is safe to ignore quota. + Handles(operation admission.Attributes) bool + // Matches returns true if the specified quota matches the input item + Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error) + // MatchingScopes takes the input specified list of scopes and input object and returns the set of scopes that matches input object. + MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) + // UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes. It returns the scopes which are in limited scopes but dont have a corresponding covering quota scope + UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) + // MatchingResources takes the input specified list of resources and returns the set of resources evaluator matches. + MatchingResources(input []corev1.ResourceName) []corev1.ResourceName + // Usage returns the resource usage for the specified object + Usage(item runtime.Object) (corev1.ResourceList, error) + // UsageStats calculates latest observed usage stats for all objects + UsageStats(options UsageStatsOptions) (UsageStats, error) +} + +// Configuration defines how the quota system is configured. +type Configuration interface { + // IgnoredResources are ignored by quota. + IgnoredResources() map[schema.GroupResource]struct{} + // Evaluators for quota evaluation. + Evaluators() []Evaluator +} + +// Registry maintains a list of evaluators +type Registry interface { + // Add to registry + Add(e Evaluator) + // Remove from registry + Remove(e Evaluator) + // Get by group resource + Get(gr schema.GroupResource) Evaluator + // List from registry + List() []Evaluator +} + +// ListerForResourceFunc knows how to get a lister for a specific resource +type ListerForResourceFunc func(schema.GroupVersionResource) (cache.GenericLister, error) diff --git a/pkg/quota/v1/resources.go b/pkg/quota/v1/resources.go new file mode 100644 index 000000000..3c2927d73 --- /dev/null +++ b/pkg/quota/v1/resources.go @@ -0,0 +1,293 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" +) + +// Equals returns true if the two lists are equivalent +func Equals(a corev1.ResourceList, b corev1.ResourceList) bool { + if len(a) != len(b) { + return false + } + + for key, value1 := range a { + value2, found := b[key] + if !found { + return false + } + if value1.Cmp(value2) != 0 { + return false + } + } + + return true +} + +// LessThanOrEqual returns true if a < b for each key in b +// If false, it returns the keys in a that exceeded b +func LessThanOrEqual(a corev1.ResourceList, b corev1.ResourceList) (bool, []corev1.ResourceName) { + result := true + resourceNames := []corev1.ResourceName{} + for key, value := range b { + if other, found := a[key]; found { + if other.Cmp(value) > 0 { + result = false + resourceNames = append(resourceNames, key) + } + } + } + return result, resourceNames +} + +// Max returns the result of Max(a, b) for each named resource +func Max(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList { + result := corev1.ResourceList{} + for key, value := range a { + if other, found := b[key]; found { + if value.Cmp(other) <= 0 { + result[key] = other.DeepCopy() + continue + } + } + result[key] = value.DeepCopy() + } + for key, value := range b { + if _, found := result[key]; !found { + result[key] = value.DeepCopy() + } + } + return result +} + +// Add returns the result of a + b for each named resource +func Add(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList { + result := corev1.ResourceList{} + for key, value := range a { + quantity := value.DeepCopy() + if other, found := b[key]; found { + quantity.Add(other) + } + result[key] = quantity + } + for key, value := range b { + if _, found := result[key]; !found { + result[key] = value.DeepCopy() + } + } + return result +} + +// SubtractWithNonNegativeResult - subtracts and returns result of a - b but +// makes sure we don't return negative values to prevent negative resource usage. +func SubtractWithNonNegativeResult(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList { + zero := resource.MustParse("0") + + result := corev1.ResourceList{} + for key, value := range a { + quantity := value.DeepCopy() + if other, found := b[key]; found { + quantity.Sub(other) + } + if quantity.Cmp(zero) > 0 { + result[key] = quantity + } else { + result[key] = zero + } + } + + for key := range b { + if _, found := result[key]; !found { + result[key] = zero + } + } + return result +} + +// Subtract returns the result of a - b for each named resource +func Subtract(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList { + result := corev1.ResourceList{} + for key, value := range a { + quantity := value.DeepCopy() + if other, found := b[key]; found { + quantity.Sub(other) + } + result[key] = quantity + } + for key, value := range b { + if _, found := result[key]; !found { + quantity := value.DeepCopy() + quantity.Neg() + result[key] = quantity + } + } + return result +} + +// Mask returns a new resource list that only has the values with the specified names +func Mask(resources corev1.ResourceList, names []corev1.ResourceName) corev1.ResourceList { + nameSet := ToSet(names) + result := corev1.ResourceList{} + for key, value := range resources { + if nameSet.Has(string(key)) { + result[key] = value.DeepCopy() + } + } + return result +} + +// ResourceNames returns a list of all resource names in the ResourceList +func ResourceNames(resources corev1.ResourceList) []corev1.ResourceName { + result := []corev1.ResourceName{} + for resourceName := range resources { + result = append(result, resourceName) + } + return result +} + +// Contains returns true if the specified item is in the list of items +func Contains(items []corev1.ResourceName, item corev1.ResourceName) bool { + for _, i := range items { + if i == item { + return true + } + } + return false +} + +// ContainsPrefix returns true if the specified item has a prefix that contained in given prefix Set +func ContainsPrefix(prefixSet []string, item corev1.ResourceName) bool { + for _, prefix := range prefixSet { + if strings.HasPrefix(string(item), prefix) { + return true + } + } + return false +} + +// Intersection returns the intersection of both list of resources, deduped and sorted +func Intersection(a []corev1.ResourceName, b []corev1.ResourceName) []corev1.ResourceName { + result := make([]corev1.ResourceName, 0, len(a)) + for _, item := range a { + if Contains(result, item) { + continue + } + if !Contains(b, item) { + continue + } + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { return result[i] < result[j] }) + return result +} + +// Difference returns the list of resources resulting from a-b, deduped and sorted +func Difference(a []corev1.ResourceName, b []corev1.ResourceName) []corev1.ResourceName { + result := make([]corev1.ResourceName, 0, len(a)) + for _, item := range a { + if Contains(b, item) || Contains(result, item) { + continue + } + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { return result[i] < result[j] }) + return result +} + +// IsZero returns true if each key maps to the quantity value 0 +func IsZero(a corev1.ResourceList) bool { + zero := resource.MustParse("0") + for _, v := range a { + if v.Cmp(zero) != 0 { + return false + } + } + return true +} + +// IsNegative returns the set of resource names that have a negative value. +func IsNegative(a corev1.ResourceList) []corev1.ResourceName { + results := []corev1.ResourceName{} + zero := resource.MustParse("0") + for k, v := range a { + if v.Cmp(zero) < 0 { + results = append(results, k) + } + } + return results +} + +// ToSet takes a list of resource names and converts to a string set +func ToSet(resourceNames []corev1.ResourceName) sets.String { + result := sets.NewString() + for _, resourceName := range resourceNames { + result.Insert(string(resourceName)) + } + return result +} + +// CalculateUsage calculates and returns the requested ResourceList usage. +// If an error is returned, usage only contains the resources which encountered no calculation errors. +func CalculateUsage(namespaceName string, scopes []corev1.ResourceQuotaScope, hardLimits corev1.ResourceList, registry Registry, scopeSelector *corev1.ScopeSelector) (corev1.ResourceList, error) { + // find the intersection between the hard resources on the quota + // and the resources this controller can track to know what we can + // look to measure updated usage stats for + hardResources := ResourceNames(hardLimits) + potentialResources := []corev1.ResourceName{} + evaluators := registry.List() + for _, evaluator := range evaluators { + potentialResources = append(potentialResources, evaluator.MatchingResources(hardResources)...) + } + // NOTE: the intersection just removes duplicates since the evaluator match intersects with hard + matchedResources := Intersection(hardResources, potentialResources) + + errors := []error{} + + // sum the observed usage from each evaluator + newUsage := corev1.ResourceList{} + for _, evaluator := range evaluators { + // only trigger the evaluator if it matches a resource in the quota, otherwise, skip calculating anything + intersection := evaluator.MatchingResources(matchedResources) + if len(intersection) == 0 { + continue + } + + usageStatsOptions := UsageStatsOptions{Namespace: namespaceName, Scopes: scopes, Resources: intersection, ScopeSelector: scopeSelector} + stats, err := evaluator.UsageStats(usageStatsOptions) + if err != nil { + // remember the error + errors = append(errors, err) + // exclude resources which encountered calculation errors + matchedResources = Difference(matchedResources, intersection) + continue + } + newUsage = Add(newUsage, stats.Used) + } + + // mask the observed usage to only the set of resources tracked by this quota + // merge our observed usage with the quota usage status + // if the new usage is different than the last usage, we will need to do an update + newUsage = Mask(newUsage, matchedResources) + return newUsage, utilerrors.NewAggregate(errors) +} diff --git a/pkg/quota/v1/resources_test.go b/pkg/quota/v1/resources_test.go new file mode 100644 index 000000000..b8c85a157 --- /dev/null +++ b/pkg/quota/v1/resources_test.go @@ -0,0 +1,412 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestEquals(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + b corev1.ResourceList + expected bool + }{ + "isEqual": { + a: corev1.ResourceList{}, + b: corev1.ResourceList{}, + expected: true, + }, + "isEqualWithKeys": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + b: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: true, + }, + "isNotEqualSameKeys": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + b: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: false, + }, + "isNotEqualDiffKeys": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + b: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("1"), + }, + expected: false, + }, + } + for testName, testCase := range testCases { + if result := Equals(testCase.a, testCase.b); result != testCase.expected { + t.Errorf("%s expected: %v, actual: %v, a=%v, b=%v", testName, testCase.expected, result, testCase.a, testCase.b) + } + } +} + +func TestMax(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + b corev1.ResourceList + expected corev1.ResourceList + }{ + "noKeys": { + a: corev1.ResourceList{}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{}, + }, + "toEmpty": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + "matching": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("150m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("150m")}, + }, + "matching(reverse)": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("150m")}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("150m")}, + }, + "matching-equal": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + } + for testName, testCase := range testCases { + sum := Max(testCase.a, testCase.b) + if result := Equals(testCase.expected, sum); !result { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, sum) + } + } +} + +func TestAdd(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + b corev1.ResourceList + expected corev1.ResourceList + }{ + "noKeys": { + a: corev1.ResourceList{}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{}, + }, + "toEmpty": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + "matching": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}, + }, + } + for testName, testCase := range testCases { + sum := Add(testCase.a, testCase.b) + if result := Equals(testCase.expected, sum); !result { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, sum) + } + } +} + +func TestSubtract(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + b corev1.ResourceList + expected corev1.ResourceList + }{ + "noKeys": { + a: corev1.ResourceList{}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{}, + }, + "value-empty": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + b: corev1.ResourceList{}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + "empty-value": { + a: corev1.ResourceList{}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("-100m")}, + }, + "value-value": { + a: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}, + b: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + expected: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + }, + } + for testName, testCase := range testCases { + sub := Subtract(testCase.a, testCase.b) + if result := Equals(testCase.expected, sub); !result { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, sub) + } + } +} + +func TestResourceNames(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + expected []corev1.ResourceName + }{ + "empty": { + a: corev1.ResourceList{}, + expected: []corev1.ResourceName{}, + }, + "values": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: []corev1.ResourceName{corev1.ResourceMemory, corev1.ResourceCPU}, + }, + } + for testName, testCase := range testCases { + actualSet := ToSet(ResourceNames(testCase.a)) + expectedSet := ToSet(testCase.expected) + if !actualSet.Equal(expectedSet) { + t.Errorf("%s expected: %v, actual: %v", testName, expectedSet, actualSet) + } + } +} + +func TestContains(t *testing.T) { + testCases := map[string]struct { + a []corev1.ResourceName + b corev1.ResourceName + expected bool + }{ + "does-not-contain": { + a: []corev1.ResourceName{corev1.ResourceMemory}, + b: corev1.ResourceCPU, + expected: false, + }, + "does-contain": { + a: []corev1.ResourceName{corev1.ResourceMemory, corev1.ResourceCPU}, + b: corev1.ResourceCPU, + expected: true, + }, + } + for testName, testCase := range testCases { + if actual := Contains(testCase.a, testCase.b); actual != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, actual) + } + } +} + +func TestContainsPrefix(t *testing.T) { + testCases := map[string]struct { + a []string + b corev1.ResourceName + expected bool + }{ + "does-not-contain": { + a: []string{corev1.ResourceHugePagesPrefix}, + b: corev1.ResourceCPU, + expected: false, + }, + "does-contain": { + a: []string{corev1.ResourceHugePagesPrefix}, + b: corev1.ResourceName(corev1.ResourceHugePagesPrefix + "2Mi"), + expected: true, + }, + } + for testName, testCase := range testCases { + if actual := ContainsPrefix(testCase.a, testCase.b); actual != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, actual) + } + } +} + +func TestIsZero(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + expected bool + }{ + "empty": { + a: corev1.ResourceList{}, + expected: true, + }, + "zero": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + expected: true, + }, + "non-zero": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: false, + }, + } + for testName, testCase := range testCases { + if result := IsZero(testCase.a); result != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, result) + } + } +} + +func TestIsNegative(t *testing.T) { + testCases := map[string]struct { + a corev1.ResourceList + expected []corev1.ResourceName + }{ + "empty": { + a: corev1.ResourceList{}, + expected: []corev1.ResourceName{}, + }, + "some-negative": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("-10"), + corev1.ResourceMemory: resource.MustParse("0"), + }, + expected: []corev1.ResourceName{corev1.ResourceCPU}, + }, + "all-negative": { + a: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("-200m"), + corev1.ResourceMemory: resource.MustParse("-1Gi"), + }, + expected: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + }, + } + for testName, testCase := range testCases { + actual := IsNegative(testCase.a) + actualSet := ToSet(actual) + expectedSet := ToSet(testCase.expected) + if !actualSet.Equal(expectedSet) { + t.Errorf("%s expected: %v, actual: %v", testName, expectedSet, actualSet) + } + } +} + +func TestIntersection(t *testing.T) { + testCases := map[string]struct { + a []corev1.ResourceName + b []corev1.ResourceName + expected []corev1.ResourceName + }{ + "empty": { + a: []corev1.ResourceName{}, + b: []corev1.ResourceName{}, + expected: []corev1.ResourceName{}, + }, + "equal": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + expected: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + }, + "a has extra": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU}, + expected: []corev1.ResourceName{corev1.ResourceCPU}, + }, + "b has extra": { + a: []corev1.ResourceName{corev1.ResourceCPU}, + b: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + expected: []corev1.ResourceName{corev1.ResourceCPU}, + }, + "dedupes": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU}, + expected: []corev1.ResourceName{corev1.ResourceCPU}, + }, + "sorts": { + a: []corev1.ResourceName{corev1.ResourceMemory, corev1.ResourceMemory, corev1.ResourceCPU, corev1.ResourceCPU}, + b: []corev1.ResourceName{corev1.ResourceMemory, corev1.ResourceMemory, corev1.ResourceCPU, corev1.ResourceCPU}, + expected: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + }, + } + for testName, testCase := range testCases { + actual := Intersection(testCase.a, testCase.b) + if !reflect.DeepEqual(actual, testCase.expected) { + t.Errorf("%s expected: %#v, actual: %#v", testName, testCase.expected, actual) + } + } +} + +func TestDifference(t *testing.T) { + testCases := map[string]struct { + a []corev1.ResourceName + b []corev1.ResourceName + expected []corev1.ResourceName + }{ + "empty": { + a: []corev1.ResourceName{}, + b: []corev1.ResourceName{}, + expected: []corev1.ResourceName{}, + }, + "equal": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + expected: []corev1.ResourceName{}, + }, + "a has extra": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU}, + expected: []corev1.ResourceName{corev1.ResourceMemory}, + }, + "b has extra": { + a: []corev1.ResourceName{corev1.ResourceCPU}, + b: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + expected: []corev1.ResourceName{}, + }, + "dedupes": { + a: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourceMemory}, + b: []corev1.ResourceName{corev1.ResourceCPU}, + expected: []corev1.ResourceName{corev1.ResourceMemory}, + }, + "sorts": { + a: []corev1.ResourceName{corev1.ResourceMemory, corev1.ResourceMemory, corev1.ResourceCPU, corev1.ResourceCPU}, + b: []corev1.ResourceName{}, + expected: []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}, + }, + } + for testName, testCase := range testCases { + actual := Difference(testCase.a, testCase.b) + if !reflect.DeepEqual(actual, testCase.expected) { + t.Errorf("%s expected: %#v, actual: %#v", testName, testCase.expected, actual) + } + } +}