Merge pull request #2999 from ctripcloud/update-resolver

Support Connection to ResourceInterpretWebhook without DNS Service

cmd/agent/app/agent.go

@@ -11,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/informers"
 	kubeclientset "k8s.io/client-go/kubernetes"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/term"
@@ -229,7 +230,16 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
 	restConfig := mgr.GetConfig()
 	dynamicClientSet := dynamic.NewForConfigOrDie(restConfig)
 	controlPlaneInformerManager := genericmanager.NewSingleClusterInformerManager(dynamicClientSet, 0, stopChan)
-	resourceInterpreter := resourceinterpreter.NewResourceInterpreter(controlPlaneInformerManager)
+	controlPlaneKubeClientSet := kubeclientset.NewForConfigOrDie(restConfig)
+
+	// We need a service lister to build a resource interpreter with `ClusterIPServiceResolver`
+	// witch allows connection to the customized interpreter webhook without a cluster DNS service.
+	sharedFactory := informers.NewSharedInformerFactory(controlPlaneKubeClientSet, 0)
+	serviceLister := sharedFactory.Core().V1().Services().Lister()
+	sharedFactory.Start(stopChan)
+	sharedFactory.WaitForCacheSync(stopChan)
+
+	resourceInterpreter := resourceinterpreter.NewResourceInterpreter(controlPlaneInformerManager, serviceLister)
 	if err := mgr.Add(resourceInterpreter); err != nil {
 		return fmt.Errorf("failed to setup custom resource interpreter: %w", err)
 	}

cmd/controller-manager/app/controllermanager.go

@@ -12,6 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/informers"
 	kubeclientset "k8s.io/client-go/kubernetes"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/term"
@@ -531,6 +532,7 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
 	restConfig := mgr.GetConfig()
 	dynamicClientSet := dynamic.NewForConfigOrDie(restConfig)
 	discoverClientSet := discovery.NewDiscoveryClientForConfigOrDie(restConfig)
+	kubeClientSet := kubeclientset.NewForConfigOrDie(restConfig)
 
 	overrideManager := overridemanager.New(mgr.GetClient(), mgr.GetEventRecorderFor(overridemanager.OverrideManagerName))
 	skippedResourceConfig := util.NewSkippedResourceConfig()
@@ -541,7 +543,14 @@ func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stop
 
 	controlPlaneInformerManager := genericmanager.NewSingleClusterInformerManager(dynamicClientSet, 0, stopChan)
 
-	resourceInterpreter := resourceinterpreter.NewResourceInterpreter(controlPlaneInformerManager)
+	// We need a service lister to build a resource interpreter with `ClusterIPServiceResolver`
+	// witch allows connection to the customized interpreter webhook without a cluster DNS service.
+	sharedFactory := informers.NewSharedInformerFactory(kubeClientSet, 0)
+	serviceLister := sharedFactory.Core().V1().Services().Lister()
+	sharedFactory.Start(stopChan)
+	sharedFactory.WaitForCacheSync(stopChan)
+
+	resourceInterpreter := resourceinterpreter.NewResourceInterpreter(controlPlaneInformerManager, serviceLister)
 	if err := mgr.Add(resourceInterpreter); err != nil {
 		klog.Fatalf("Failed to setup custom resource interpreter: %v", err)
 	}

pkg/resourceinterpreter/customized/webhook/customized.go

@@ -13,7 +13,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
+	corev1 "k8s.io/client-go/listers/core/v1"
 	"k8s.io/klog/v2"
+	"k8s.io/kube-aggregator/pkg/apiserver"
 	utiltrace "k8s.io/utils/trace"
 
 	configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1"
@@ -33,7 +35,7 @@ type CustomizedInterpreter struct {
 }
 
 // NewCustomizedInterpreter return a new CustomizedInterpreter.
-func NewCustomizedInterpreter(informer genericmanager.SingleClusterInformerManager) (*CustomizedInterpreter, error) {
+func NewCustomizedInterpreter(informer genericmanager.SingleClusterInformerManager, serviceLister corev1.ServiceLister) (*CustomizedInterpreter, error) {
 	cm, err := webhookutil.NewClientManager(
 		[]schema.GroupVersion{configv1alpha1.SchemeGroupVersion},
 		configv1alpha1.AddToScheme,
@@ -45,8 +47,9 @@ func NewCustomizedInterpreter(informer genericmanager.SingleClusterInformerManag
 	if err != nil {
 		return nil, err
 	}
+
 	cm.SetAuthenticationInfoResolver(authInfoResolver)
-	cm.SetServiceResolver(webhookutil.NewDefaultServiceResolver())
+	cm.SetServiceResolver(apiserver.NewClusterIPServiceResolver(serviceLister))
 
 	return &CustomizedInterpreter{
 		hookManager:   configmanager.NewExploreConfigManager(informer),

pkg/resourceinterpreter/interpreter.go

@@ -6,6 +6,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	corev1 "k8s.io/client-go/listers/core/v1"
 	"k8s.io/klog/v2"
 
 	configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1"
@@ -51,14 +52,16 @@ type ResourceInterpreter interface {
 }
 
 // NewResourceInterpreter builds a new ResourceInterpreter object.
-func NewResourceInterpreter(informer genericmanager.SingleClusterInformerManager) ResourceInterpreter {
+func NewResourceInterpreter(informer genericmanager.SingleClusterInformerManager, serviceLister corev1.ServiceLister) ResourceInterpreter {
 	return &customResourceInterpreterImpl{
-		informer: informer,
+		informer:      informer,
+		serviceLister: serviceLister,
 	}
 }
 
 type customResourceInterpreterImpl struct {
-	informer genericmanager.SingleClusterInformerManager
+	informer      genericmanager.SingleClusterInformerManager
+	serviceLister corev1.ServiceLister
 
 	configurableInterpreter *declarative.ConfigurableInterpreter
 	customizedInterpreter   *webhook.CustomizedInterpreter
@@ -70,7 +73,7 @@ type customResourceInterpreterImpl struct {
 func (i *customResourceInterpreterImpl) Start(ctx context.Context) (err error) {
 	klog.Infof("Starting custom resource interpreter.")
 
-	i.customizedInterpreter, err = webhook.NewCustomizedInterpreter(i.informer)
+	i.customizedInterpreter, err = webhook.NewCustomizedInterpreter(i.informer, i.serviceLister)
 	if err != nil {
 		return
 	}

vendor/k8s.io/apimachinery/pkg/api/meta/table/table.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package table
+
+import (
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/duration"
+)
+
+// MetaToTableRow converts a list or object into one or more table rows. The provided rowFn is invoked for
+// each accessed item, with name and age being passed to each.
+func MetaToTableRow(obj runtime.Object, rowFn func(obj runtime.Object, m metav1.Object, name, age string) ([]interface{}, error)) ([]metav1.TableRow, error) {
+	if meta.IsListType(obj) {
+		rows := make([]metav1.TableRow, 0, 16)
+		err := meta.EachListItem(obj, func(obj runtime.Object) error {
+			nestedRows, err := MetaToTableRow(obj, rowFn)
+			if err != nil {
+				return err
+			}
+			rows = append(rows, nestedRows...)
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		return rows, nil
+	}
+
+	rows := make([]metav1.TableRow, 0, 1)
+	m, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, err
+	}
+	row := metav1.TableRow{
+		Object: runtime.RawExtension{Object: obj},
+	}
+	row.Cells, err = rowFn(obj, m, m.GetName(), ConvertToHumanReadableDateType(m.GetCreationTimestamp()))
+	if err != nil {
+		return nil, err
+	}
+	rows = append(rows, row)
+	return rows, nil
+}
+
+// ConvertToHumanReadableDateType returns the elapsed time since timestamp in
+// human-readable approximation.
+func ConvertToHumanReadableDateType(timestamp metav1.Time) string {
+	if timestamp.IsZero() {
+		return "<unknown>"
+	}
+	return duration.HumanDuration(time.Since(timestamp.Time))
+}

vendor/k8s.io/apiserver/pkg/util/proxy/proxy.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxy
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/url"
+	"strconv"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	listersv1 "k8s.io/client-go/listers/core/v1"
+)
+
+// findServicePort finds the service port by name or numerically.
+func findServicePort(svc *v1.Service, port int32) (*v1.ServicePort, error) {
+	for _, svcPort := range svc.Spec.Ports {
+		if svcPort.Port == port {
+			return &svcPort, nil
+		}
+	}
+	return nil, errors.NewServiceUnavailable(fmt.Sprintf("no service port %d found for service %q", port, svc.Name))
+}
+
+// ResourceLocation returns a URL to which one can send traffic for the specified service.
+func ResolveEndpoint(services listersv1.ServiceLister, endpoints listersv1.EndpointsLister, namespace, id string, port int32) (*url.URL, error) {
+	svc, err := services.Services(namespace).Get(id)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case svc.Spec.Type == v1.ServiceTypeClusterIP, svc.Spec.Type == v1.ServiceTypeLoadBalancer, svc.Spec.Type == v1.ServiceTypeNodePort:
+		// these are fine
+	default:
+		return nil, fmt.Errorf("unsupported service type %q", svc.Spec.Type)
+	}
+
+	svcPort, err := findServicePort(svc, port)
+	if err != nil {
+		return nil, err
+	}
+
+	eps, err := endpoints.Endpoints(namespace).Get(svc.Name)
+	if err != nil {
+		return nil, err
+	}
+	if len(eps.Subsets) == 0 {
+		return nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", svc.Name))
+	}
+
+	// Pick a random Subset to start searching from.
+	ssSeed := rand.Intn(len(eps.Subsets))
+
+	// Find a Subset that has the port.
+	for ssi := 0; ssi < len(eps.Subsets); ssi++ {
+		ss := &eps.Subsets[(ssSeed+ssi)%len(eps.Subsets)]
+		if len(ss.Addresses) == 0 {
+			continue
+		}
+		for i := range ss.Ports {
+			if ss.Ports[i].Name == svcPort.Name {
+				// Pick a random address.
+				ip := ss.Addresses[rand.Intn(len(ss.Addresses))].IP
+				port := int(ss.Ports[i].Port)
+				return &url.URL{
+					Scheme: "https",
+					Host:   net.JoinHostPort(ip, strconv.Itoa(port)),
+				}, nil
+			}
+		}
+	}
+	return nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", id))
+}
+
+func ResolveCluster(services listersv1.ServiceLister, namespace, id string, port int32) (*url.URL, error) {
+	svc, err := services.Services(namespace).Get(id)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case svc.Spec.Type == v1.ServiceTypeClusterIP && svc.Spec.ClusterIP == v1.ClusterIPNone:
+		return nil, fmt.Errorf(`cannot route to service with ClusterIP "None"`)
+	// use IP from a clusterIP for these service types
+	case svc.Spec.Type == v1.ServiceTypeClusterIP, svc.Spec.Type == v1.ServiceTypeLoadBalancer, svc.Spec.Type == v1.ServiceTypeNodePort:
+		svcPort, err := findServicePort(svc, port)
+		if err != nil {
+			return nil, err
+		}
+		return &url.URL{
+			Scheme: "https",
+			Host:   net.JoinHostPort(svc.Spec.ClusterIP, fmt.Sprintf("%d", svcPort.Port)),
+		}, nil
+	case svc.Spec.Type == v1.ServiceTypeExternalName:
+		return &url.URL{
+			Scheme: "https",
+			Host:   net.JoinHostPort(svc.Spec.ExternalName, fmt.Sprintf("%d", port)),
+		}, nil
+	default:
+		return nil, fmt.Errorf("unsupported service type %q", svc.Spec.Type)
+	}
+}

vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package install
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+)
+
+// Install registers the API group and adds types to a scheme
+func Install(scheme *runtime.Scheme) {
+	utilruntime.Must(apiregistration.AddToScheme(scheme))
+	utilruntime.Must(v1.AddToScheme(scheme))
+	utilruntime.Must(v1beta1.AddToScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion))
+}

vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation/validation.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/validation"
+	"k8s.io/apimachinery/pkg/api/validation/path"
+	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+)
+
+// ValidateAPIService validates that the APIService is correctly defined.
+func ValidateAPIService(apiService *apiregistration.APIService) field.ErrorList {
+	requiredName := apiService.Spec.Version + "." + apiService.Spec.Group
+
+	allErrs := validation.ValidateObjectMeta(&apiService.ObjectMeta, false,
+		func(name string, prefix bool) []string {
+			if minimalFailures := path.IsValidPathSegmentName(name); len(minimalFailures) > 0 {
+				return minimalFailures
+			}
+			// the name *must* be version.group
+			if name != requiredName {
+				return []string{fmt.Sprintf("must be `spec.version+\".\"+spec.group`: %q", requiredName)}
+			}
+
+			return []string{}
+		},
+		field.NewPath("metadata"))
+
+	// in this case we allow empty group
+	if len(apiService.Spec.Group) == 0 && apiService.Spec.Version != "v1" {
+		allErrs = append(allErrs, field.Required(field.NewPath("spec", "group"), "only v1 may have an empty group and it better be legacy kube"))
+	}
+	if len(apiService.Spec.Group) > 0 {
+		for _, errString := range utilvalidation.IsDNS1123Subdomain(apiService.Spec.Group) {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "group"), apiService.Spec.Group, errString))
+		}
+	}
+
+	for _, errString := range utilvalidation.IsDNS1035Label(apiService.Spec.Version) {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), apiService.Spec.Version, errString))
+	}
+
+	if apiService.Spec.GroupPriorityMinimum <= 0 || apiService.Spec.GroupPriorityMinimum > 20000 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "groupPriorityMinimum"), apiService.Spec.GroupPriorityMinimum, "must be positive and less than 20000"))
+	}
+	if apiService.Spec.VersionPriority <= 0 || apiService.Spec.VersionPriority > 1000 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "versionPriority"), apiService.Spec.VersionPriority, "must be positive and less than 1000"))
+	}
+
+	if apiService.Spec.Service == nil {
+		if len(apiService.Spec.CABundle) != 0 {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "caBundle"), fmt.Sprintf("%d bytes", len(apiService.Spec.CABundle)), "local APIServices may not have a caBundle"))
+		}
+		if apiService.Spec.InsecureSkipTLSVerify {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "insecureSkipTLSVerify"), apiService.Spec.InsecureSkipTLSVerify, "local APIServices may not have insecureSkipTLSVerify"))
+		}
+		return allErrs
+	}
+
+	if len(apiService.Spec.Service.Namespace) == 0 {
+		allErrs = append(allErrs, field.Required(field.NewPath("spec", "service", "namespace"), ""))
+	}
+	if len(apiService.Spec.Service.Name) == 0 {
+		allErrs = append(allErrs, field.Required(field.NewPath("spec", "service", "name"), ""))
+	}
+	if errs := utilvalidation.IsValidPortNum(int(apiService.Spec.Service.Port)); errs != nil {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "service", "port"), apiService.Spec.Service.Port, "port is not valid: "+strings.Join(errs, ", ")))
+	}
+	if apiService.Spec.InsecureSkipTLSVerify && len(apiService.Spec.CABundle) > 0 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "insecureSkipTLSVerify"), apiService.Spec.InsecureSkipTLSVerify, "may not be true if caBundle is present"))
+	}
+
+	return allErrs
+}
+
+// ValidateAPIServiceUpdate validates an update of APIService.
+func ValidateAPIServiceUpdate(newAPIService *apiregistration.APIService, oldAPIService *apiregistration.APIService) field.ErrorList {
+	allErrs := validation.ValidateObjectMetaUpdate(&newAPIService.ObjectMeta, &oldAPIService.ObjectMeta, field.NewPath("metadata"))
+	allErrs = append(allErrs, ValidateAPIService(newAPIService)...)
+
+	return allErrs
+}
+
+// ValidateAPIServiceStatus validates that the APIService status is one of 'True', 'False' or 'Unknown'.
+func ValidateAPIServiceStatus(status *apiregistration.APIServiceStatus, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	for i, condition := range status.Conditions {
+		if condition.Status != apiregistration.ConditionTrue &&
+			condition.Status != apiregistration.ConditionFalse &&
+			condition.Status != apiregistration.ConditionUnknown {
+			allErrs = append(allErrs, field.NotSupported(fldPath.Child("conditions").Index(i).Child("status"), condition.Status, []string{
+				string(apiregistration.ConditionTrue), string(apiregistration.ConditionFalse), string(apiregistration.ConditionUnknown)}))
+		}
+	}
+
+	return allErrs
+}
+
+// ValidateAPIServiceStatusUpdate validates an update of the status field of APIService.
+func ValidateAPIServiceStatusUpdate(newAPIService *apiregistration.APIService, oldAPIService *apiregistration.APIService) field.ErrorList {
+	allErrs := validation.ValidateObjectMetaUpdate(&newAPIService.ObjectMeta, &oldAPIService.ObjectMeta, field.NewPath("metadata"))
+	allErrs = append(allErrs, ValidateAPIServiceStatus(&newAPIService.Status, field.NewPath("status"))...)
+	return allErrs
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go

@@ -0,0 +1,564 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/version"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
+
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	"k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions"
+	listers "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+	openapicontroller "k8s.io/kube-aggregator/pkg/controllers/openapi"
+	openapiaggregator "k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator"
+	openapiv3controller "k8s.io/kube-aggregator/pkg/controllers/openapiv3"
+	openapiv3aggregator "k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator"
+	statuscontrollers "k8s.io/kube-aggregator/pkg/controllers/status"
+	apiservicerest "k8s.io/kube-aggregator/pkg/registry/apiservice/rest"
+)
+
+func init() {
+	// we need to add the options (like ListOptions) to empty v1
+	metav1.AddToGroupVersion(aggregatorscheme.Scheme, schema.GroupVersion{Group: "", Version: "v1"})
+
+	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
+	aggregatorscheme.Scheme.AddUnversionedTypes(unversioned,
+		&metav1.Status{},
+		&metav1.APIVersions{},
+		&metav1.APIGroupList{},
+		&metav1.APIGroup{},
+		&metav1.APIResourceList{},
+	)
+}
+
+const (
+	// legacyAPIServiceName is the fixed name of the only non-groupified API version
+	legacyAPIServiceName = "v1."
+	// StorageVersionPostStartHookName is the name of the storage version updater post start hook.
+	StorageVersionPostStartHookName = "built-in-resources-storage-version-updater"
+)
+
+// ExtraConfig represents APIServices-specific configuration
+type ExtraConfig struct {
+	// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
+	// this to confirm the proxy's identity
+	ProxyClientCertFile string
+	ProxyClientKeyFile  string
+
+	// If present, the Dial method will be used for dialing out to delegate
+	// apiservers.
+	ProxyTransport *http.Transport
+
+	// Mechanism by which the Aggregator will resolve services. Required.
+	ServiceResolver ServiceResolver
+
+	RejectForwardingRedirects bool
+}
+
+// Config represents the configuration needed to create an APIAggregator.
+type Config struct {
+	GenericConfig *genericapiserver.RecommendedConfig
+	ExtraConfig   ExtraConfig
+}
+
+type completedConfig struct {
+	GenericConfig genericapiserver.CompletedConfig
+	ExtraConfig   *ExtraConfig
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+type runnable interface {
+	Run(stopCh <-chan struct{}) error
+}
+
+// preparedGenericAPIServer is a private wrapper that enforces a call of PrepareRun() before Run can be invoked.
+type preparedAPIAggregator struct {
+	*APIAggregator
+	runnable runnable
+}
+
+// APIAggregator contains state for a Kubernetes cluster master/api server.
+type APIAggregator struct {
+	GenericAPIServer *genericapiserver.GenericAPIServer
+
+	// provided for easier embedding
+	APIRegistrationInformers informers.SharedInformerFactory
+
+	delegateHandler http.Handler
+
+	// proxyCurrentCertKeyContent holds he client cert used to identify this proxy. Backing APIServices use this to confirm the proxy's identity
+	proxyCurrentCertKeyContent certKeyFunc
+	proxyTransport             *http.Transport
+
+	// proxyHandlers are the proxy handlers that are currently registered, keyed by apiservice.name
+	proxyHandlers map[string]*proxyHandler
+	// handledGroups are the groups that already have routes
+	handledGroups sets.String
+
+	// lister is used to add group handling for /apis/<group> aggregator lookups based on
+	// controller state
+	lister listers.APIServiceLister
+
+	// Information needed to determine routing for the aggregator
+	serviceResolver ServiceResolver
+
+	// Enable swagger and/or OpenAPI if these configs are non-nil.
+	openAPIConfig *openapicommon.Config
+
+	// Enable OpenAPI V3 if these configs are non-nil
+	openAPIV3Config *openapicommon.Config
+
+	// openAPIAggregationController downloads and merges OpenAPI v2 specs.
+	openAPIAggregationController *openapicontroller.AggregationController
+
+	// openAPIV3AggregationController downloads and caches OpenAPI v3 specs.
+	openAPIV3AggregationController *openapiv3controller.AggregationController
+
+	// discoveryAggregationController downloads and caches discovery documents
+	// from all aggregated apiservices so they are available from /apis endpoint
+	// when discovery with resources are requested
+	discoveryAggregationController DiscoveryAggregationController
+
+	// egressSelector selects the proper egress dialer to communicate with the custom apiserver
+	// overwrites proxyTransport dialer if not nil
+	egressSelector *egressselector.EgressSelector
+
+	// rejectForwardingRedirects is whether to allow to forward redirect response
+	rejectForwardingRedirects bool
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (cfg *Config) Complete() CompletedConfig {
+	c := completedConfig{
+		cfg.GenericConfig.Complete(),
+		&cfg.ExtraConfig,
+	}
+
+	// the kube aggregator wires its own discovery mechanism
+	// TODO eventually collapse this by extracting all of the discovery out
+	c.GenericConfig.EnableDiscovery = false
+	version := version.Get()
+	c.GenericConfig.Version = &version
+
+	return CompletedConfig{&c}
+}
+
+// NewWithDelegate returns a new instance of APIAggregator from the given config.
+func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.DelegationTarget) (*APIAggregator, error) {
+	genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget)
+	if err != nil {
+		return nil, err
+	}
+
+	apiregistrationClient, err := clientset.NewForConfig(c.GenericConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	informerFactory := informers.NewSharedInformerFactory(
+		apiregistrationClient,
+		5*time.Minute, // this is effectively used as a refresh interval right now.  Might want to do something nicer later on.
+	)
+
+	// apiServiceRegistrationControllerInitiated is closed when APIServiceRegistrationController has finished "installing" all known APIServices.
+	// At this point we know that the proxy handler knows about APIServices and can handle client requests.
+	// Before it might have resulted in a 404 response which could have serious consequences for some controllers like  GC and NS
+	//
+	// Note that the APIServiceRegistrationController waits for APIServiceInformer to synced before doing its work.
+	apiServiceRegistrationControllerInitiated := make(chan struct{})
+	if err := genericServer.RegisterMuxAndDiscoveryCompleteSignal("APIServiceRegistrationControllerInitiated", apiServiceRegistrationControllerInitiated); err != nil {
+		return nil, err
+	}
+
+	s := &APIAggregator{
+		GenericAPIServer:           genericServer,
+		delegateHandler:            delegationTarget.UnprotectedHandler(),
+		proxyTransport:             c.ExtraConfig.ProxyTransport,
+		proxyHandlers:              map[string]*proxyHandler{},
+		handledGroups:              sets.String{},
+		lister:                     informerFactory.Apiregistration().V1().APIServices().Lister(),
+		APIRegistrationInformers:   informerFactory,
+		serviceResolver:            c.ExtraConfig.ServiceResolver,
+		openAPIConfig:              c.GenericConfig.OpenAPIConfig,
+		openAPIV3Config:            c.GenericConfig.OpenAPIV3Config,
+		egressSelector:             c.GenericConfig.EgressSelector,
+		proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
+		rejectForwardingRedirects:  c.ExtraConfig.RejectForwardingRedirects,
+	}
+
+	// used later  to filter the served resource by those that have expired.
+	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter, resourceExpirationEvaluator.ShouldServeForVersion(1, 22))
+	if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
+		return nil, err
+	}
+
+	enabledVersions := sets.NewString()
+	for v := range apiGroupInfo.VersionedResourcesStorageMap {
+		enabledVersions.Insert(v)
+	}
+	if !enabledVersions.Has(v1.SchemeGroupVersion.Version) {
+		return nil, fmt.Errorf("API group/version %s must be enabled", v1.SchemeGroupVersion.String())
+	}
+
+	apisHandler := &apisHandler{
+		codecs:         aggregatorscheme.Codecs,
+		lister:         s.lister,
+		discoveryGroup: discoveryGroup(enabledVersions),
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+		apisHandlerWithAggregationSupport := aggregated.WrapAggregatedDiscoveryToHandler(apisHandler, s.GenericAPIServer.AggregatedDiscoveryGroupManager)
+		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandlerWithAggregationSupport)
+	} else {
+		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler)
+	}
+	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler)
+
+	apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().V1().APIServices(), s)
+	if len(c.ExtraConfig.ProxyClientCertFile) > 0 && len(c.ExtraConfig.ProxyClientKeyFile) > 0 {
+		aggregatorProxyCerts, err := dynamiccertificates.NewDynamicServingContentFromFiles("aggregator-proxy-cert", c.ExtraConfig.ProxyClientCertFile, c.ExtraConfig.ProxyClientKeyFile)
+		if err != nil {
+			return nil, err
+		}
+		// We are passing the context to ProxyCerts.RunOnce as it needs to implement RunOnce(ctx) however the
+		// context is not used at all. So passing a empty context shouldn't be a problem
+		ctx := context.TODO()
+		if err := aggregatorProxyCerts.RunOnce(ctx); err != nil {
+			return nil, err
+		}
+		aggregatorProxyCerts.AddListener(apiserviceRegistrationController)
+		s.proxyCurrentCertKeyContent = aggregatorProxyCerts.CurrentCertKeyContent
+
+		s.GenericAPIServer.AddPostStartHookOrDie("aggregator-reload-proxy-client-cert", func(postStartHookContext genericapiserver.PostStartHookContext) error {
+			// generate a context  from stopCh. This is to avoid modifying files which are relying on apiserver
+			// TODO: See if we can pass ctx to the current method
+			ctx, cancel := context.WithCancel(context.Background())
+			go func() {
+				select {
+				case <-postStartHookContext.StopCh:
+					cancel() // stopCh closed, so cancel our context
+				case <-ctx.Done():
+				}
+			}()
+			go aggregatorProxyCerts.Run(ctx, 1)
+			return nil
+		})
+	}
+
+	availableController, err := statuscontrollers.NewAvailableConditionController(
+		informerFactory.Apiregistration().V1().APIServices(),
+		c.GenericConfig.SharedInformerFactory.Core().V1().Services(),
+		c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(),
+		apiregistrationClient.ApiregistrationV1(),
+		c.ExtraConfig.ProxyTransport,
+		(func() ([]byte, []byte))(s.proxyCurrentCertKeyContent),
+		s.serviceResolver,
+		c.GenericConfig.EgressSelector,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	s.GenericAPIServer.AddPostStartHookOrDie("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {
+		informerFactory.Start(context.StopCh)
+		c.GenericConfig.SharedInformerFactory.Start(context.StopCh)
+		return nil
+	})
+	s.GenericAPIServer.AddPostStartHookOrDie("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error {
+		go apiserviceRegistrationController.Run(context.StopCh, apiServiceRegistrationControllerInitiated)
+		select {
+		case <-context.StopCh:
+		case <-apiServiceRegistrationControllerInitiated:
+		}
+
+		return nil
+	})
+	s.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-available-controller", func(context genericapiserver.PostStartHookContext) error {
+		// if we end up blocking for long periods of time, we may need to increase workers.
+		go availableController.Run(5, context.StopCh)
+		return nil
+	})
+
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
+		utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
+		// Spawn a goroutine in aggregator apiserver to update storage version for
+		// all built-in resources
+		s.GenericAPIServer.AddPostStartHookOrDie(StorageVersionPostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error {
+			// Wait for apiserver-identity to exist first before updating storage
+			// versions, to avoid storage version GC accidentally garbage-collecting
+			// storage versions.
+			kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
+			if err != nil {
+				return err
+			}
+			if err := wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
+				_, err := kubeClient.CoordinationV1().Leases(metav1.NamespaceSystem).Get(
+					context.TODO(), s.GenericAPIServer.APIServerID, metav1.GetOptions{})
+				if apierrors.IsNotFound(err) {
+					return false, nil
+				}
+				if err != nil {
+					return false, err
+				}
+				return true, nil
+			}, hookContext.StopCh); err != nil {
+				return fmt.Errorf("failed to wait for apiserver-identity lease %s to be created: %v",
+					s.GenericAPIServer.APIServerID, err)
+			}
+			// Technically an apiserver only needs to update storage version once during bootstrap.
+			// Reconcile StorageVersion objects every 10 minutes will help in the case that the
+			// StorageVersion objects get accidentally modified/deleted by a different agent. In that
+			// case, the reconciliation ensures future storage migration still works. If nothing gets
+			// changed, the reconciliation update is a noop and gets short-circuited by the apiserver,
+			// therefore won't change the resource version and trigger storage migration.
+			go wait.PollImmediateUntil(10*time.Minute, func() (bool, error) {
+				// All apiservers (aggregator-apiserver, kube-apiserver, apiextensions-apiserver)
+				// share the same generic apiserver config. The same StorageVersion manager is used
+				// to register all built-in resources when the generic apiservers install APIs.
+				s.GenericAPIServer.StorageVersionManager.UpdateStorageVersions(hookContext.LoopbackClientConfig, s.GenericAPIServer.APIServerID)
+				return false, nil
+			}, hookContext.StopCh)
+			// Once the storage version updater finishes the first round of update,
+			// the PostStartHook will return to unblock /healthz. The handler chain
+			// won't block write requests anymore. Check every second since it's not
+			// expensive.
+			wait.PollImmediateUntil(1*time.Second, func() (bool, error) {
+				return s.GenericAPIServer.StorageVersionManager.Completed(), nil
+			}, hookContext.StopCh)
+			return nil
+		})
+	}
+
+	return s, nil
+}
+
+// PrepareRun prepares the aggregator to run, by setting up the OpenAPI spec &
+// aggregated discovery document and calling the generic PrepareRun.
+func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) {
+	// add post start hook before generic PrepareRun in order to be before /healthz installation
+	if s.openAPIConfig != nil {
+		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-openapi-controller", func(context genericapiserver.PostStartHookContext) error {
+			go s.openAPIAggregationController.Run(context.StopCh)
+			return nil
+		})
+	}
+
+	if s.openAPIV3Config != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.OpenAPIV3) {
+		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-openapiv3-controller", func(context genericapiserver.PostStartHookContext) error {
+			go s.openAPIV3AggregationController.Run(context.StopCh)
+			return nil
+		})
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+		s.discoveryAggregationController = NewDiscoveryManager(
+			s.GenericAPIServer.AggregatedDiscoveryGroupManager,
+		)
+
+		// Setup discovery endpoint
+		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-discovery-controller", func(context genericapiserver.PostStartHookContext) error {
+			// Run discovery manager's worker to watch for new/removed/updated
+			// APIServices to the discovery document can be updated at runtime
+			go s.discoveryAggregationController.Run(context.StopCh)
+			return nil
+		})
+	}
+
+	prepared := s.GenericAPIServer.PrepareRun()
+
+	// delay OpenAPI setup until the delegate had a chance to setup their OpenAPI handlers
+	if s.openAPIConfig != nil {
+		specDownloader := openapiaggregator.NewDownloader()
+		openAPIAggregator, err := openapiaggregator.BuildAndRegisterAggregator(
+			&specDownloader,
+			s.GenericAPIServer.NextDelegate(),
+			s.GenericAPIServer.Handler.GoRestfulContainer.RegisteredWebServices(),
+			s.openAPIConfig,
+			s.GenericAPIServer.Handler.NonGoRestfulMux)
+		if err != nil {
+			return preparedAPIAggregator{}, err
+		}
+		s.openAPIAggregationController = openapicontroller.NewAggregationController(&specDownloader, openAPIAggregator)
+	}
+
+	if s.openAPIV3Config != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.OpenAPIV3) {
+		specDownloaderV3 := openapiv3aggregator.NewDownloader()
+		openAPIV3Aggregator, err := openapiv3aggregator.BuildAndRegisterAggregator(
+			specDownloaderV3,
+			s.GenericAPIServer.NextDelegate(),
+			s.GenericAPIServer.Handler.NonGoRestfulMux)
+		if err != nil {
+			return preparedAPIAggregator{}, err
+		}
+		s.openAPIV3AggregationController = openapiv3controller.NewAggregationController(openAPIV3Aggregator)
+	}
+
+	return preparedAPIAggregator{APIAggregator: s, runnable: prepared}, nil
+}
+
+func (s preparedAPIAggregator) Run(stopCh <-chan struct{}) error {
+	return s.runnable.Run(stopCh)
+}
+
+// AddAPIService adds an API service.  It is not thread-safe, so only call it on one thread at a time please.
+// It's a slow moving API, so its ok to run the controller on a single thread
+func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
+	// if the proxyHandler already exists, it needs to be updated. The aggregation bits do not
+	// since they are wired against listers because they require multiple resources to respond
+	if proxyHandler, exists := s.proxyHandlers[apiService.Name]; exists {
+		proxyHandler.updateAPIService(apiService)
+		if s.openAPIAggregationController != nil {
+			s.openAPIAggregationController.UpdateAPIService(proxyHandler, apiService)
+		}
+		if s.openAPIV3AggregationController != nil {
+			s.openAPIV3AggregationController.UpdateAPIService(proxyHandler, apiService)
+		}
+		// Forward calls to discovery manager to update discovery document
+		if s.discoveryAggregationController != nil {
+			handlerCopy := *proxyHandler
+			handlerCopy.setServiceAvailable(true)
+			s.discoveryAggregationController.AddAPIService(apiService, &handlerCopy)
+		}
+		return nil
+	}
+
+	proxyPath := "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
+	// v1. is a special case for the legacy API.  It proxies to a wider set of endpoints.
+	if apiService.Name == legacyAPIServiceName {
+		proxyPath = "/api"
+	}
+
+	// register the proxy handler
+	proxyHandler := &proxyHandler{
+		localDelegate:              s.delegateHandler,
+		proxyCurrentCertKeyContent: s.proxyCurrentCertKeyContent,
+		proxyTransport:             s.proxyTransport,
+		serviceResolver:            s.serviceResolver,
+		egressSelector:             s.egressSelector,
+		rejectForwardingRedirects:  s.rejectForwardingRedirects,
+	}
+	proxyHandler.updateAPIService(apiService)
+	if s.openAPIAggregationController != nil {
+		s.openAPIAggregationController.AddAPIService(proxyHandler, apiService)
+	}
+	if s.openAPIV3AggregationController != nil {
+		s.openAPIV3AggregationController.AddAPIService(proxyHandler, apiService)
+	}
+	if s.discoveryAggregationController != nil {
+		s.discoveryAggregationController.AddAPIService(apiService, proxyHandler)
+	}
+
+	s.proxyHandlers[apiService.Name] = proxyHandler
+	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(proxyPath, proxyHandler)
+	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandlePrefix(proxyPath+"/", proxyHandler)
+
+	// if we're dealing with the legacy group, we're done here
+	if apiService.Name == legacyAPIServiceName {
+		return nil
+	}
+
+	// if we've already registered the path with the handler, we don't want to do it again.
+	if s.handledGroups.Has(apiService.Spec.Group) {
+		return nil
+	}
+
+	// it's time to register the group aggregation endpoint
+	groupPath := "/apis/" + apiService.Spec.Group
+	groupDiscoveryHandler := &apiGroupHandler{
+		codecs:    aggregatorscheme.Codecs,
+		groupName: apiService.Spec.Group,
+		lister:    s.lister,
+		delegate:  s.delegateHandler,
+	}
+	// aggregation is protected
+	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(groupPath, groupDiscoveryHandler)
+	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle(groupPath+"/", groupDiscoveryHandler)
+	s.handledGroups.Insert(apiService.Spec.Group)
+	return nil
+}
+
+// RemoveAPIService removes the APIService from being handled.  It is not thread-safe, so only call it on one thread at a time please.
+// It's a slow moving API, so it's ok to run the controller on a single thread.
+func (s *APIAggregator) RemoveAPIService(apiServiceName string) {
+	// Forward calls to discovery manager to update discovery document
+	if s.discoveryAggregationController != nil {
+		s.discoveryAggregationController.RemoveAPIService(apiServiceName)
+	}
+
+	version := v1helper.APIServiceNameToGroupVersion(apiServiceName)
+
+	proxyPath := "/apis/" + version.Group + "/" + version.Version
+	// v1. is a special case for the legacy API.  It proxies to a wider set of endpoints.
+	if apiServiceName == legacyAPIServiceName {
+		proxyPath = "/api"
+	}
+	s.GenericAPIServer.Handler.NonGoRestfulMux.Unregister(proxyPath)
+	s.GenericAPIServer.Handler.NonGoRestfulMux.Unregister(proxyPath + "/")
+	if s.openAPIAggregationController != nil {
+		s.openAPIAggregationController.RemoveAPIService(apiServiceName)
+	}
+	if s.openAPIV3AggregationController != nil {
+		s.openAPIAggregationController.RemoveAPIService(apiServiceName)
+	}
+	delete(s.proxyHandlers, apiServiceName)
+
+	// TODO unregister group level discovery when there are no more versions for the group
+	// We don't need this right away because the handler properly delegates when no versions are present
+}
+
+// DefaultAPIResourceConfigSource returns default configuration for an APIResource.
+func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
+	ret := serverstorage.NewResourceConfig()
+	// NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list.
+	ret.EnableVersions(
+		v1.SchemeGroupVersion,
+		v1beta1.SchemeGroupVersion,
+	)
+
+	return ret
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go

@@ -0,0 +1,209 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"fmt"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
+	listers "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers"
+)
+
+// APIHandlerManager defines the behaviour that an API handler should have.
+type APIHandlerManager interface {
+	AddAPIService(apiService *v1.APIService) error
+	RemoveAPIService(apiServiceName string)
+}
+
+// APIServiceRegistrationController is responsible for registering and removing API services.
+type APIServiceRegistrationController struct {
+	apiHandlerManager APIHandlerManager
+
+	apiServiceLister listers.APIServiceLister
+	apiServiceSynced cache.InformerSynced
+
+	// To allow injection for testing.
+	syncFn func(key string) error
+
+	queue workqueue.RateLimitingInterface
+}
+
+var _ dynamiccertificates.Listener = &APIServiceRegistrationController{}
+
+// NewAPIServiceRegistrationController returns a new APIServiceRegistrationController.
+func NewAPIServiceRegistrationController(apiServiceInformer informers.APIServiceInformer, apiHandlerManager APIHandlerManager) *APIServiceRegistrationController {
+	c := &APIServiceRegistrationController{
+		apiHandlerManager: apiHandlerManager,
+		apiServiceLister:  apiServiceInformer.Lister(),
+		apiServiceSynced:  apiServiceInformer.Informer().HasSynced,
+		queue:             workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "APIServiceRegistrationController"),
+	}
+
+	apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addAPIService,
+		UpdateFunc: c.updateAPIService,
+		DeleteFunc: c.deleteAPIService,
+	})
+
+	c.syncFn = c.sync
+
+	return c
+}
+
+func (c *APIServiceRegistrationController) sync(key string) error {
+	apiService, err := c.apiServiceLister.Get(key)
+	if apierrors.IsNotFound(err) {
+		c.apiHandlerManager.RemoveAPIService(key)
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	return c.apiHandlerManager.AddAPIService(apiService)
+}
+
+// Run starts APIServiceRegistrationController which will process all registration requests until stopCh is closed.
+func (c *APIServiceRegistrationController) Run(stopCh <-chan struct{}, handlerSyncedCh chan<- struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting APIServiceRegistrationController")
+	defer klog.Info("Shutting down APIServiceRegistrationController")
+
+	if !controllers.WaitForCacheSync("APIServiceRegistrationController", stopCh, c.apiServiceSynced) {
+		return
+	}
+
+	/// initially sync all APIServices to make sure the proxy handler is complete
+	if err := wait.PollImmediateUntil(time.Second, func() (bool, error) {
+		services, err := c.apiServiceLister.List(labels.Everything())
+		if err != nil {
+			utilruntime.HandleError(fmt.Errorf("failed to initially list APIServices: %v", err))
+			return false, nil
+		}
+		for _, s := range services {
+			if err := c.apiHandlerManager.AddAPIService(s); err != nil {
+				utilruntime.HandleError(fmt.Errorf("failed to initially sync APIService %s: %v", s.Name, err))
+				return false, nil
+			}
+		}
+		return true, nil
+	}, stopCh); err == wait.ErrWaitTimeout {
+		utilruntime.HandleError(fmt.Errorf("timed out waiting for proxy handler to initialize"))
+		return
+	} else if err != nil {
+		panic(fmt.Errorf("unexpected error: %v", err))
+	}
+	close(handlerSyncedCh)
+
+	// only start one worker thread since its a slow moving API and the aggregation server adding bits
+	// aren't threadsafe
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+func (c *APIServiceRegistrationController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *APIServiceRegistrationController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	err := c.syncFn(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err))
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *APIServiceRegistrationController) enqueueInternal(obj *v1.APIService) {
+	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+	if err != nil {
+		klog.Errorf("Couldn't get key for object %#v: %v", obj, err)
+		return
+	}
+
+	c.queue.Add(key)
+}
+
+func (c *APIServiceRegistrationController) addAPIService(obj interface{}) {
+	castObj := obj.(*v1.APIService)
+	klog.V(4).Infof("Adding %s", castObj.Name)
+	c.enqueueInternal(castObj)
+}
+
+func (c *APIServiceRegistrationController) updateAPIService(obj, _ interface{}) {
+	castObj := obj.(*v1.APIService)
+	klog.V(4).Infof("Updating %s", castObj.Name)
+	c.enqueueInternal(castObj)
+}
+
+func (c *APIServiceRegistrationController) deleteAPIService(obj interface{}) {
+	castObj, ok := obj.(*v1.APIService)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*v1.APIService)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	klog.V(4).Infof("Deleting %q", castObj.Name)
+	c.enqueueInternal(castObj)
+}
+
+// Enqueue queues all apiservices to be rehandled.
+// This method is used by the controller to notify when the proxy cert content changes.
+func (c *APIServiceRegistrationController) Enqueue() {
+	apiServices, err := c.apiServiceLister.List(labels.Everything())
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	for _, apiService := range apiServices {
+		c.addAPIService(apiService)
+	}
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/handler_apis.go

@@ -0,0 +1,166 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"net/http"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+
+	apiregistrationv1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+	apiregistrationv1beta1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+	listers "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+)
+
+// apisHandler serves the `/apis` endpoint.
+// This is registered as a filter so that it never collides with any explicitly registered endpoints
+type apisHandler struct {
+	codecs         serializer.CodecFactory
+	lister         listers.APIServiceLister
+	discoveryGroup metav1.APIGroup
+}
+
+func discoveryGroup(enabledVersions sets.String) metav1.APIGroup {
+	retval := metav1.APIGroup{
+		Name: apiregistrationv1api.GroupName,
+		Versions: []metav1.GroupVersionForDiscovery{
+			{
+				GroupVersion: apiregistrationv1api.SchemeGroupVersion.String(),
+				Version:      apiregistrationv1api.SchemeGroupVersion.Version,
+			},
+		},
+		PreferredVersion: metav1.GroupVersionForDiscovery{
+			GroupVersion: apiregistrationv1api.SchemeGroupVersion.String(),
+			Version:      apiregistrationv1api.SchemeGroupVersion.Version,
+		},
+	}
+
+	if enabledVersions.Has(apiregistrationv1beta1api.SchemeGroupVersion.Version) {
+		retval.Versions = append(retval.Versions, metav1.GroupVersionForDiscovery{
+			GroupVersion: apiregistrationv1beta1api.SchemeGroupVersion.String(),
+			Version:      apiregistrationv1beta1api.SchemeGroupVersion.Version,
+		})
+	}
+
+	return retval
+}
+
+func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	discoveryGroupList := &metav1.APIGroupList{
+		// always add OUR api group to the list first.  Since we'll never have a registered APIService for it
+		// and since this is the crux of the API, having this first will give our names priority.  It's good to be king.
+		Groups: []metav1.APIGroup{r.discoveryGroup},
+	}
+
+	apiServices, err := r.lister.List(labels.Everything())
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	apiServicesByGroup := apiregistrationv1apihelper.SortedByGroupAndVersion(apiServices)
+	for _, apiGroupServers := range apiServicesByGroup {
+		// skip the legacy group
+		if len(apiGroupServers[0].Spec.Group) == 0 {
+			continue
+		}
+		discoveryGroup := convertToDiscoveryAPIGroup(apiGroupServers)
+		if discoveryGroup != nil {
+			discoveryGroupList.Groups = append(discoveryGroupList.Groups, *discoveryGroup)
+		}
+	}
+
+	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroupList, false)
+}
+
+// convertToDiscoveryAPIGroup takes apiservices in a single group and returns a discovery compatible object.
+// if none of the services are available, it will return nil.
+func convertToDiscoveryAPIGroup(apiServices []*apiregistrationv1api.APIService) *metav1.APIGroup {
+	apiServicesByGroup := apiregistrationv1apihelper.SortedByGroupAndVersion(apiServices)[0]
+
+	var discoveryGroup *metav1.APIGroup
+
+	for _, apiService := range apiServicesByGroup {
+		// the first APIService which is valid becomes the default
+		if discoveryGroup == nil {
+			discoveryGroup = &metav1.APIGroup{
+				Name: apiService.Spec.Group,
+				PreferredVersion: metav1.GroupVersionForDiscovery{
+					GroupVersion: apiService.Spec.Group + "/" + apiService.Spec.Version,
+					Version:      apiService.Spec.Version,
+				},
+			}
+		}
+
+		discoveryGroup.Versions = append(discoveryGroup.Versions,
+			metav1.GroupVersionForDiscovery{
+				GroupVersion: apiService.Spec.Group + "/" + apiService.Spec.Version,
+				Version:      apiService.Spec.Version,
+			},
+		)
+	}
+
+	return discoveryGroup
+}
+
+// apiGroupHandler serves the `/apis/<group>` endpoint.
+type apiGroupHandler struct {
+	codecs    serializer.CodecFactory
+	groupName string
+
+	lister listers.APIServiceLister
+
+	delegate http.Handler
+}
+
+func (r *apiGroupHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	apiServices, err := r.lister.List(labels.Everything())
+	if statusErr, ok := err.(*apierrors.StatusError); ok {
+		responsewriters.WriteRawJSON(int(statusErr.Status().Code), statusErr.Status(), w)
+		return
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	apiServicesForGroup := []*apiregistrationv1api.APIService{}
+	for _, apiService := range apiServices {
+		if apiService.Spec.Group == r.groupName {
+			apiServicesForGroup = append(apiServicesForGroup, apiService)
+		}
+	}
+
+	if len(apiServicesForGroup) == 0 {
+		r.delegate.ServeHTTP(w, req)
+		return
+	}
+
+	discoveryGroup := convertToDiscoveryAPIGroup(apiServicesForGroup)
+	if discoveryGroup == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroup, false)
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/handler_discovery.go

@@ -0,0 +1,577 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints"
+	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	scheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+)
+
+var APIRegistrationGroupVersion metav1.GroupVersion = metav1.GroupVersion{Group: "apiregistration.k8s.io", Version: "v1"}
+
+// Maximum is 20000. Set to higher than that so apiregistration always is listed
+// first (mirrors v1 discovery behavior)
+var APIRegistrationGroupPriority int = 20001
+
+// Given a list of APIServices and proxyHandlers for contacting them,
+// DiscoveryManager caches a list of discovery documents for each server
+
+type DiscoveryAggregationController interface {
+	// Adds or Updates an APIService from the Aggregated Discovery Controller's
+	// knowledge base
+	// Thread-safe
+	AddAPIService(apiService *apiregistrationv1.APIService, handler http.Handler)
+
+	// Removes an APIService from the Aggregated Discovery Controller's Knowledge
+	// bank
+	// Thread-safe
+	RemoveAPIService(apiServiceName string)
+
+	// Spwans a worker which waits for added/updated apiservices and updates
+	// the unified discovery document by contacting the aggregated api services
+	Run(stopCh <-chan struct{})
+
+	// Returns true if all non-local APIServices that have been added
+	// are synced at least once to the discovery document
+	ExternalServicesSynced() bool
+}
+
+type discoveryManager struct {
+	// Locks `services`
+	servicesLock sync.RWMutex
+
+	// Map from APIService's name (or a unique string for local servers)
+	// to information about contacting that API Service
+	apiServices map[string]groupVersionInfo
+
+	// Locks cachedResults
+	resultsLock sync.RWMutex
+
+	// Map from APIService.Spec.Service to the previously fetched value
+	// (Note that many APIServices might use the same APIService.Spec.Service)
+	cachedResults map[serviceKey]cachedResult
+
+	// Queue of dirty apiServiceKey which need to be refreshed
+	// It is important that the reconciler for this queue does not excessively
+	// contact the apiserver if a key was enqueued before the server was last
+	// contacted.
+	dirtyAPIServiceQueue workqueue.RateLimitingInterface
+
+	// Merged handler which stores all known groupversions
+	mergedDiscoveryHandler discoveryendpoint.ResourceManager
+}
+
+// Version of Service/Spec with relevant fields for use as a cache key
+type serviceKey struct {
+	Namespace string
+	Name      string
+	Port      int32
+}
+
+// Human-readable String representation used for logs
+func (s serviceKey) String() string {
+	return fmt.Sprintf("%v/%v:%v", s.Namespace, s.Name, s.Port)
+}
+
+func newServiceKey(service apiregistrationv1.ServiceReference) serviceKey {
+	// Docs say. Defaults to 443 for compatibility reasons.
+	// BETA: Should this be a shared constant to avoid drifting with the
+	// implementation?
+	port := int32(443)
+	if service.Port != nil {
+		port = *service.Port
+	}
+
+	return serviceKey{
+		Name:      service.Name,
+		Namespace: service.Namespace,
+		Port:      port,
+	}
+}
+
+type cachedResult struct {
+	// Currently cached discovery document for this service
+	// Map from group name to version name to
+	discovery map[metav1.GroupVersion]apidiscoveryv2beta1.APIVersionDiscovery
+
+	// ETag hash of the cached discoveryDocument
+	etag string
+
+	// Guaranteed to be a time less than the time the server responded with the
+	// discovery data.
+	lastUpdated time.Time
+}
+
+// Information about a specific APIService/GroupVersion
+type groupVersionInfo struct {
+	// Date this APIService was marked dirty.
+	// Guaranteed to be a time greater than the most recent time the APIService
+	// was known to be modified.
+	//
+	// Used for request deduplication to ensure the data used to reconcile each
+	// apiservice was retrieved after the time of the APIService change:
+	// real_apiservice_change_time < groupVersionInfo.lastMarkedDirty < cachedResult.lastUpdated < real_document_fresh_time
+	//
+	// This ensures that if the apiservice was changed after the last cached entry
+	// was stored, the discovery document will always be re-fetched.
+	lastMarkedDirty time.Time
+
+	// Last time sync function was run for this GV.
+	lastReconciled time.Time
+
+	// ServiceReference of this GroupVersion. This identifies the Service which
+	// describes how to contact the server responsible for this GroupVersion.
+	service serviceKey
+
+	// groupPriority describes the priority of the APIService's group for sorting
+	groupPriority int
+
+	// groupPriority describes the priority of the APIService version for sorting
+	versionPriority int
+
+	// Method for contacting the service
+	handler http.Handler
+}
+
+var _ DiscoveryAggregationController = &discoveryManager{}
+
+func NewDiscoveryManager(
+	target discoveryendpoint.ResourceManager,
+) DiscoveryAggregationController {
+	return &discoveryManager{
+		mergedDiscoveryHandler: target,
+		apiServices:            make(map[string]groupVersionInfo),
+		cachedResults:          make(map[serviceKey]cachedResult),
+		dirtyAPIServiceQueue:   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "discovery-manager"),
+	}
+}
+
+// Returns discovery data for the given apiservice.
+// Caches the result.
+// Returns the cached result if it is retrieved after the apiservice was last
+// marked dirty
+// If there was an error in fetching, returns the stale cached result if it exists,
+// and a non-nil error
+// If the result is current, returns nil error and non-nil result
+func (dm *discoveryManager) fetchFreshDiscoveryForService(gv metav1.GroupVersion, info groupVersionInfo) (*cachedResult, error) {
+	// Lookup last cached result for this apiservice's service.
+	cached, exists := dm.getCacheEntryForService(info.service)
+
+	// If entry exists and was updated after the given time, just stop now
+	if exists && cached.lastUpdated.After(info.lastMarkedDirty) {
+		return &cached, nil
+	}
+
+	// If we have a handler to contact the server for this APIService, and
+	// the cache entry is too old to use, refresh the cache entry now.
+	handler := http.TimeoutHandler(info.handler, 5*time.Second, "request timed out")
+	req, err := http.NewRequest("GET", "/apis", nil)
+	if err != nil {
+		// NewRequest should not fail, but if it does for some reason,
+		// log it and continue
+		return &cached, fmt.Errorf("failed to create http.Request: %v", err)
+	}
+
+	// Apply aggregator user to request
+	req = req.WithContext(
+		request.WithUser(
+			req.Context(), &user.DefaultInfo{Name: "system:kube-aggregator", Groups: []string{"system:masters"}}))
+	req = req.WithContext(request.WithRequestInfo(req.Context(), &request.RequestInfo{
+		Path:              req.URL.Path,
+		IsResourceRequest: false,
+	}))
+	req.Header.Add("Accept", runtime.ContentTypeJSON+";g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList")
+
+	if exists && len(cached.etag) > 0 {
+		req.Header.Add("If-None-Match", cached.etag)
+	}
+
+	// Important that the time recorded in the data's "lastUpdated" is conservatively
+	// from BEFORE the request is dispatched so that lastUpdated can be used to
+	// de-duplicate requests.
+	now := time.Now()
+	writer := newInMemoryResponseWriter()
+	handler.ServeHTTP(writer, req)
+
+	switch writer.respCode {
+	case http.StatusNotModified:
+		// Keep old entry, update timestamp
+		cached = cachedResult{
+			discovery:   cached.discovery,
+			etag:        cached.etag,
+			lastUpdated: now,
+		}
+
+		dm.setCacheEntryForService(info.service, cached)
+		return &cached, nil
+	case http.StatusNotAcceptable:
+		// Discovery Document is not being served at all.
+		// Fall back to legacy discovery information
+		if len(gv.Version) == 0 {
+			return nil, errors.New("not found")
+		}
+
+		var path string
+		if len(gv.Group) == 0 {
+			path = "/api/" + gv.Version
+		} else {
+			path = "/apis/" + gv.Group + "/" + gv.Version
+		}
+
+		req, err := http.NewRequest("GET", path, nil)
+		if err != nil {
+			// NewRequest should not fail, but if it does for some reason,
+			// log it and continue
+			return nil, fmt.Errorf("failed to create http.Request: %v", err)
+		}
+
+		// Apply aggregator user to request
+		req = req.WithContext(
+			request.WithUser(
+				req.Context(), &user.DefaultInfo{Name: "system:kube-aggregator"}))
+
+		// req.Header.Add("Accept", runtime.ContentTypeProtobuf)
+		req.Header.Add("Accept", runtime.ContentTypeJSON)
+
+		if exists && len(cached.etag) > 0 {
+			req.Header.Add("If-None-Match", cached.etag)
+		}
+
+		writer := newInMemoryResponseWriter()
+		handler.ServeHTTP(writer, req)
+
+		if writer.respCode != http.StatusOK {
+			return nil, fmt.Errorf("failed to download discovery for %s: %v", path, writer.String())
+		}
+
+		parsed := &metav1.APIResourceList{}
+		if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), writer.data, parsed); err != nil {
+			return nil, err
+		}
+
+		// Create a discomap with single group-version
+		resources, err := endpoints.ConvertGroupVersionIntoToDiscovery(parsed.APIResources)
+		if err != nil {
+			return nil, err
+		}
+
+		discoMap := map[metav1.GroupVersion]apidiscoveryv2beta1.APIVersionDiscovery{
+			// Convert old-style APIGroupList to new information
+			gv: {
+				Version:   gv.Version,
+				Resources: resources,
+			},
+		}
+
+		cached = cachedResult{
+			discovery:   discoMap,
+			lastUpdated: now,
+		}
+
+		// Save the resolve, because it is still useful in case other services
+		// are already marked dirty. THey can use it without making http request
+		dm.setCacheEntryForService(info.service, cached)
+		return &cached, nil
+
+	case http.StatusOK:
+		parsed := &apidiscoveryv2beta1.APIGroupDiscoveryList{}
+		if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), writer.data, parsed); err != nil {
+			return nil, err
+		}
+		klog.V(3).Infof("DiscoveryManager: Successfully downloaded discovery for %s", info.service.String())
+
+		// Convert discovery info into a map for convenient lookup later
+		discoMap := map[metav1.GroupVersion]apidiscoveryv2beta1.APIVersionDiscovery{}
+		for _, g := range parsed.Items {
+			for _, v := range g.Versions {
+				discoMap[metav1.GroupVersion{Group: g.Name, Version: v.Version}] = v
+			}
+		}
+
+		// Save cached result
+		cached = cachedResult{
+			discovery:   discoMap,
+			etag:        writer.Header().Get("Etag"),
+			lastUpdated: now,
+		}
+		dm.setCacheEntryForService(info.service, cached)
+		return &cached, nil
+
+	default:
+		klog.Infof("DiscoveryManager: Failed to download discovery for %v: %v %s",
+			info.service.String(), writer.respCode, writer.data)
+		return nil, fmt.Errorf("service %s returned non-success response code: %v",
+			info.service.String(), writer.respCode)
+	}
+}
+
+// Try to sync a single APIService.
+func (dm *discoveryManager) syncAPIService(apiServiceName string) error {
+	info, exists := dm.getInfoForAPIService(apiServiceName)
+
+	gv := helper.APIServiceNameToGroupVersion(apiServiceName)
+	mgv := metav1.GroupVersion{Group: gv.Group, Version: gv.Version}
+
+	if !exists {
+		// apiservice was removed. remove it from merged discovery
+		dm.mergedDiscoveryHandler.RemoveGroupVersion(mgv)
+		return nil
+	}
+
+	// Lookup last cached result for this apiservice's service.
+	now := time.Now()
+	cached, err := dm.fetchFreshDiscoveryForService(mgv, info)
+
+	info.lastReconciled = now
+	dm.setInfoForAPIService(apiServiceName, &info)
+
+	var entry apidiscoveryv2beta1.APIVersionDiscovery
+
+	// Extract the APIService's specific resource information from the
+	// groupversion
+	if cached == nil {
+		// There was an error fetching discovery for this APIService, and
+		// there is nothing in the cache for this GV.
+		//
+		// Just use empty GV to mark that GV exists, but no resources.
+		// Also mark that it is stale to indicate the fetch failed
+		// TODO: Maybe also stick in a status for the version the error?
+		entry = apidiscoveryv2beta1.APIVersionDiscovery{
+			Version: gv.Version,
+		}
+	} else {
+		// Find our specific groupversion within the discovery document
+		entry, exists = cached.discovery[mgv]
+		if exists {
+			// The stale/fresh entry has our GV, so we can include it in the doc
+		} else {
+			// Successfully fetched discovery information from the server, but
+			// the server did not include this groupversion?
+			entry = apidiscoveryv2beta1.APIVersionDiscovery{
+				Version: gv.Version,
+			}
+		}
+	}
+
+	// The entry's staleness depends upon if `fetchFreshDiscoveryForService`
+	// returned an error or not.
+	if err == nil {
+		entry.Freshness = apidiscoveryv2beta1.DiscoveryFreshnessCurrent
+	} else {
+		entry.Freshness = apidiscoveryv2beta1.DiscoveryFreshnessStale
+	}
+
+	dm.mergedDiscoveryHandler.AddGroupVersion(gv.Group, entry)
+	dm.mergedDiscoveryHandler.SetGroupVersionPriority(metav1.GroupVersion(gv), info.groupPriority, info.versionPriority)
+	return nil
+}
+
+// Spwans a goroutune which waits for added/updated apiservices and updates
+// the discovery document accordingly
+func (dm *discoveryManager) Run(stopCh <-chan struct{}) {
+	klog.Info("Starting ResourceDiscoveryManager")
+
+	// Shutdown the queue since stopCh was signalled
+	defer dm.dirtyAPIServiceQueue.ShutDown()
+
+	// Spawn workers
+	// These workers wait for APIServices to be marked dirty.
+	// Worker ensures the cached discovery document hosted by the ServiceReference of
+	// the APIService is at least as fresh as the APIService, then includes the
+	// APIService's groupversion into the merged document
+	for i := 0; i < 2; i++ {
+		go func() {
+			for {
+				next, shutdown := dm.dirtyAPIServiceQueue.Get()
+				if shutdown {
+					return
+				}
+
+				func() {
+					defer dm.dirtyAPIServiceQueue.Done(next)
+
+					if err := dm.syncAPIService(next.(string)); err != nil {
+						dm.dirtyAPIServiceQueue.AddRateLimited(next)
+					} else {
+						dm.dirtyAPIServiceQueue.Forget(next)
+					}
+				}()
+			}
+		}()
+	}
+
+	// Ensure that apiregistration.k8s.io is the first group in the discovery group.
+	dm.mergedDiscoveryHandler.SetGroupVersionPriority(APIRegistrationGroupVersion, APIRegistrationGroupPriority, 0)
+
+	wait.PollUntil(1*time.Minute, func() (done bool, err error) {
+		dm.servicesLock.Lock()
+		defer dm.servicesLock.Unlock()
+
+		now := time.Now()
+
+		// Mark all non-local APIServices as dirty
+		for key, info := range dm.apiServices {
+			info.lastMarkedDirty = now
+			dm.apiServices[key] = info
+			dm.dirtyAPIServiceQueue.Add(key)
+		}
+		return false, nil
+	}, stopCh)
+}
+
+// Adds an APIService to be tracked by the discovery manager. If the APIService
+// is already known
+func (dm *discoveryManager) AddAPIService(apiService *apiregistrationv1.APIService, handler http.Handler) {
+	// If service is nil then its information is contained by a local APIService
+	// which is has already been added to the manager.
+	if apiService.Spec.Service == nil {
+		return
+	}
+
+	// Add or update APIService record and mark it as dirty
+	dm.setInfoForAPIService(apiService.Name, &groupVersionInfo{
+		groupPriority:   int(apiService.Spec.GroupPriorityMinimum),
+		versionPriority: int(apiService.Spec.VersionPriority),
+		handler:         handler,
+		lastMarkedDirty: time.Now(),
+		service:         newServiceKey(*apiService.Spec.Service),
+	})
+	dm.dirtyAPIServiceQueue.Add(apiService.Name)
+}
+
+func (dm *discoveryManager) RemoveAPIService(apiServiceName string) {
+	if dm.setInfoForAPIService(apiServiceName, nil) != nil {
+		// mark dirty if there was actually something deleted
+		dm.dirtyAPIServiceQueue.Add(apiServiceName)
+	}
+}
+
+func (dm *discoveryManager) ExternalServicesSynced() bool {
+	dm.servicesLock.RLock()
+	defer dm.servicesLock.RUnlock()
+	for _, info := range dm.apiServices {
+		if info.lastReconciled.IsZero() {
+			return false
+		}
+	}
+
+	return true
+}
+
+//
+// Lock-protected accessors
+//
+
+func (dm *discoveryManager) getCacheEntryForService(key serviceKey) (cachedResult, bool) {
+	dm.resultsLock.RLock()
+	defer dm.resultsLock.RUnlock()
+
+	result, ok := dm.cachedResults[key]
+	return result, ok
+}
+
+func (dm *discoveryManager) setCacheEntryForService(key serviceKey, result cachedResult) {
+	dm.resultsLock.Lock()
+	defer dm.resultsLock.Unlock()
+
+	dm.cachedResults[key] = result
+}
+
+func (dm *discoveryManager) getInfoForAPIService(name string) (groupVersionInfo, bool) {
+	dm.servicesLock.RLock()
+	defer dm.servicesLock.RUnlock()
+
+	result, ok := dm.apiServices[name]
+	return result, ok
+}
+
+func (dm *discoveryManager) setInfoForAPIService(name string, result *groupVersionInfo) (oldValueIfExisted *groupVersionInfo) {
+	dm.servicesLock.Lock()
+	defer dm.servicesLock.Unlock()
+
+	if oldValue, exists := dm.apiServices[name]; exists {
+		oldValueIfExisted = &oldValue
+	}
+
+	if result != nil {
+		dm.apiServices[name] = *result
+	} else {
+		delete(dm.apiServices, name)
+	}
+
+	return oldValueIfExisted
+}
+
+// !TODO: This was copied from staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go
+// which was copied from staging/src/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/downloader.go
+// so we should find a home for this
+// inMemoryResponseWriter is a http.Writer that keep the response in memory.
+type inMemoryResponseWriter struct {
+	writeHeaderCalled bool
+	header            http.Header
+	respCode          int
+	data              []byte
+}
+
+func newInMemoryResponseWriter() *inMemoryResponseWriter {
+	return &inMemoryResponseWriter{header: http.Header{}}
+}
+
+func (r *inMemoryResponseWriter) Header() http.Header {
+	return r.header
+}
+
+func (r *inMemoryResponseWriter) WriteHeader(code int) {
+	r.writeHeaderCalled = true
+	r.respCode = code
+}
+
+func (r *inMemoryResponseWriter) Write(in []byte) (int, error) {
+	if !r.writeHeaderCalled {
+		r.WriteHeader(http.StatusOK)
+	}
+	r.data = append(r.data, in...)
+	return len(in), nil
+}
+
+func (r *inMemoryResponseWriter) String() string {
+	s := fmt.Sprintf("ResponseCode: %d", r.respCode)
+	if r.data != nil {
+		s += fmt.Sprintf(", Body: %s", string(r.data))
+	}
+	if r.header != nil {
+		s += fmt.Sprintf(", Header: %s", r.header)
+	}
+	return s
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go

@@ -0,0 +1,289 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/proxy"
+	auditinternal "k8s.io/apiserver/pkg/apis/audit"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	endpointmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/apiserver/pkg/util/x509metrics"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+	apiregistrationv1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+)
+
+const (
+	aggregatorComponent string = "aggregator"
+
+	aggregatedDiscoveryTimeout = 5 * time.Second
+)
+
+type certKeyFunc func() ([]byte, []byte)
+
+// proxyHandler provides a http.Handler which will proxy traffic to locations
+// specified by items implementing Redirector.
+type proxyHandler struct {
+	// localDelegate is used to satisfy local APIServices
+	localDelegate http.Handler
+
+	// proxyCurrentCertKeyContent holds the client cert used to identify this proxy. Backing APIServices use this to confirm the proxy's identity
+	proxyCurrentCertKeyContent certKeyFunc
+	proxyTransport             *http.Transport
+
+	// Endpoints based routing to map from cluster IP to routable IP
+	serviceResolver ServiceResolver
+
+	handlingInfo atomic.Value
+
+	// egressSelector selects the proper egress dialer to communicate with the custom apiserver
+	// overwrites proxyTransport dialer if not nil
+	egressSelector *egressselector.EgressSelector
+
+	// reject to forward redirect response
+	rejectForwardingRedirects bool
+}
+
+type proxyHandlingInfo struct {
+	// local indicates that this APIService is locally satisfied
+	local bool
+
+	// name is the name of the APIService
+	name string
+	// restConfig holds the information for building a roundtripper
+	restConfig *restclient.Config
+	// transportBuildingError is an error produced while building the transport.  If this
+	// is non-nil, it will be reported to clients.
+	transportBuildingError error
+	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
+	proxyRoundTripper http.RoundTripper
+	// serviceName is the name of the service this handler proxies to
+	serviceName string
+	// namespace is the namespace the service lives in
+	serviceNamespace string
+	// serviceAvailable indicates this APIService is available or not
+	serviceAvailable bool
+	// servicePort is the port of the service this handler proxies to
+	servicePort int32
+}
+
+func proxyError(w http.ResponseWriter, req *http.Request, error string, code int) {
+	http.Error(w, error, code)
+
+	ctx := req.Context()
+	info, ok := genericapirequest.RequestInfoFrom(ctx)
+	if !ok {
+		klog.Warning("no RequestInfo found in the context")
+		return
+	}
+	// TODO: record long-running request differently? The long-running check func does not necessarily match the one of the aggregated apiserver
+	endpointmetrics.RecordRequestTermination(req, info, aggregatorComponent, code)
+}
+
+func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	value := r.handlingInfo.Load()
+	if value == nil {
+		r.localDelegate.ServeHTTP(w, req)
+		return
+	}
+	handlingInfo := value.(proxyHandlingInfo)
+	if handlingInfo.local {
+		if r.localDelegate == nil {
+			http.Error(w, "", http.StatusNotFound)
+			return
+		}
+		r.localDelegate.ServeHTTP(w, req)
+		return
+	}
+
+	if !handlingInfo.serviceAvailable {
+		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
+		return
+	}
+
+	if handlingInfo.transportBuildingError != nil {
+		proxyError(w, req, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	user, ok := genericapirequest.UserFrom(req.Context())
+	if !ok {
+		proxyError(w, req, "missing user", http.StatusInternalServerError)
+		return
+	}
+
+	// write a new location based on the existing request pointed at the target service
+	location := &url.URL{}
+	location.Scheme = "https"
+	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName, handlingInfo.servicePort)
+	if err != nil {
+		klog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
+		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
+		return
+	}
+	location.Host = rloc.Host
+	location.Path = req.URL.Path
+	location.RawQuery = req.URL.Query().Encode()
+
+	newReq, cancelFn := newRequestForProxy(location, req)
+	defer cancelFn()
+
+	if handlingInfo.proxyRoundTripper == nil {
+		proxyError(w, req, "", http.StatusNotFound)
+		return
+	}
+
+	proxyRoundTripper := handlingInfo.proxyRoundTripper
+	upgrade := httpstream.IsUpgradeRequest(req)
+
+	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
+
+	// If we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
+	// NOT use the proxyRoundTripper.  It's a direct dial that bypasses the proxyRoundTripper.  This means that we have to
+	// attach the "correct" user headers to the request ahead of time.
+	if upgrade {
+		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
+	}
+
+	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
+	if r.rejectForwardingRedirects {
+		handler.RejectForwardingRedirects = true
+	}
+	utilflowcontrol.RequestDelegated(req.Context())
+	handler.ServeHTTP(w, newReq)
+}
+
+// newRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
+func newRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
+	newCtx := req.Context()
+	cancelFn := func() {}
+
+	if requestInfo, ok := genericapirequest.RequestInfoFrom(req.Context()); ok {
+		// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
+		// segments that we are going to proxy, we have a discovery request.
+		if !requestInfo.IsResourceRequest && len(strings.Split(strings.Trim(requestInfo.Path, "/"), "/")) == 3 {
+			// discovery requests are used by kubectl and others to determine which resources a server has.  This is a cheap call that
+			// should be fast for every aggregated apiserver.  Latency for aggregation is expected to be low (as for all extensions)
+			// so forcing a short timeout here helps responsiveness of all clients.
+			newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
+		}
+	}
+
+	// WithContext creates a shallow clone of the request with the same context.
+	newReq := req.WithContext(newCtx)
+	newReq.Header = utilnet.CloneHeader(req.Header)
+	newReq.URL = location
+	newReq.Host = location.Host
+
+	// If the original request has an audit ID, let's make sure we propagate this
+	// to the aggregated server.
+	if auditID, found := audit.AuditIDFrom(req.Context()); found {
+		newReq.Header.Set(auditinternal.HeaderAuditID, string(auditID))
+	}
+
+	return newReq, cancelFn
+}
+
+// responder implements rest.Responder for assisting a connector in writing objects or errors.
+type responder struct {
+	w http.ResponseWriter
+}
+
+// TODO this should properly handle content type negotiation
+// if the caller asked for protobuf and you write JSON bad things happen.
+func (r *responder) Object(statusCode int, obj runtime.Object) {
+	responsewriters.WriteRawJSON(statusCode, obj, r.w)
+}
+
+func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
+	http.Error(r.w, err.Error(), http.StatusServiceUnavailable)
+}
+
+// these methods provide locked access to fields
+
+// Sets serviceAvailable value on proxyHandler
+// not thread safe
+func (r *proxyHandler) setServiceAvailable(value bool) {
+	info := r.handlingInfo.Load().(proxyHandlingInfo)
+	info.serviceAvailable = true
+	r.handlingInfo.Store(info)
+}
+
+func (r *proxyHandler) updateAPIService(apiService *apiregistrationv1api.APIService) {
+	if apiService.Spec.Service == nil {
+		r.handlingInfo.Store(proxyHandlingInfo{local: true})
+		return
+	}
+
+	proxyClientCert, proxyClientKey := r.proxyCurrentCertKeyContent()
+
+	clientConfig := &restclient.Config{
+		TLSClientConfig: restclient.TLSClientConfig{
+			Insecure:   apiService.Spec.InsecureSkipTLSVerify,
+			ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
+			CertData:   proxyClientCert,
+			KeyData:    proxyClientKey,
+			CAData:     apiService.Spec.CABundle,
+		},
+	}
+	clientConfig.Wrap(x509metrics.NewDeprecatedCertificateRoundTripperWrapperConstructor(
+		x509MissingSANCounter,
+		x509InsecureSHA1Counter,
+	))
+
+	newInfo := proxyHandlingInfo{
+		name:             apiService.Name,
+		restConfig:       clientConfig,
+		serviceName:      apiService.Spec.Service.Name,
+		serviceNamespace: apiService.Spec.Service.Namespace,
+		servicePort:      *apiService.Spec.Service.Port,
+		serviceAvailable: apiregistrationv1apihelper.IsAPIServiceConditionTrue(apiService, apiregistrationv1api.Available),
+	}
+	if r.egressSelector != nil {
+		networkContext := egressselector.Cluster.AsNetworkContext()
+		var egressDialer utilnet.DialFunc
+		egressDialer, err := r.egressSelector.Lookup(networkContext)
+		if err != nil {
+			klog.Warning(err.Error())
+		} else {
+			newInfo.restConfig.Dial = egressDialer
+		}
+	} else if r.proxyTransport != nil && r.proxyTransport.DialContext != nil {
+		newInfo.restConfig.Dial = r.proxyTransport.DialContext
+	}
+	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
+	if newInfo.transportBuildingError != nil {
+		klog.Warning(newInfo.transportBuildingError.Error())
+	}
+	r.handlingInfo.Store(newInfo)
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/metrics.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+var x509MissingSANCounter = metrics.NewCounter(
+	&metrics.CounterOpts{
+		Subsystem: "kube_aggregator",
+		Namespace: "apiserver",
+		Name:      "x509_missing_san_total",
+		Help: "Counts the number of requests to servers missing SAN extension " +
+			"in their serving certificate OR the number of connection failures " +
+			"due to the lack of x509 certificate SAN extension missing " +
+			"(either/or, based on the runtime environment)",
+		StabilityLevel: metrics.ALPHA,
+	},
+)
+
+var x509InsecureSHA1Counter = metrics.NewCounter(
+	&metrics.CounterOpts{
+		Subsystem: "kube_aggregator",
+		Namespace: "apiserver",
+		Name:      "x509_insecure_sha1_total",
+		Help: "Counts the number of requests to servers with insecure SHA1 signatures " +
+			"in their serving certificate OR the number of connection failures " +
+			"due to the insecure SHA1 signatures (either/or, based on the runtime environment)",
+		StabilityLevel: metrics.ALPHA,
+	},
+)
+
+func init() {
+	legacyregistry.MustRegister(x509MissingSANCounter)
+	legacyregistry.MustRegister(x509InsecureSHA1Counter)
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/resolvers.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"net/url"
+
+	"k8s.io/apiserver/pkg/util/proxy"
+	listersv1 "k8s.io/client-go/listers/core/v1"
+)
+
+// A ServiceResolver knows how to get a URL given a service.
+type ServiceResolver interface {
+	ResolveEndpoint(namespace, name string, port int32) (*url.URL, error)
+}
+
+// NewEndpointServiceResolver returns a ServiceResolver that chooses one of the
+// service's endpoints.
+func NewEndpointServiceResolver(services listersv1.ServiceLister, endpoints listersv1.EndpointsLister) ServiceResolver {
+	return &aggregatorEndpointRouting{
+		services:  services,
+		endpoints: endpoints,
+	}
+}
+
+type aggregatorEndpointRouting struct {
+	services  listersv1.ServiceLister
+	endpoints listersv1.EndpointsLister
+}
+
+func (r *aggregatorEndpointRouting) ResolveEndpoint(namespace, name string, port int32) (*url.URL, error) {
+	return proxy.ResolveEndpoint(r.services, r.endpoints, namespace, name, port)
+}
+
+// NewClusterIPServiceResolver returns a ServiceResolver that directly calls the
+// service's cluster IP.
+func NewClusterIPServiceResolver(services listersv1.ServiceLister) ServiceResolver {
+	return &aggregatorClusterRouting{
+		services: services,
+	}
+}
+
+type aggregatorClusterRouting struct {
+	services listersv1.ServiceLister
+}
+
+func (r *aggregatorClusterRouting) ResolveEndpoint(namespace, name string, port int32) (*url.URL, error) {
+	return proxy.ResolveCluster(r.services, namespace, name, port)
+}
+
+// NewLoopbackServiceResolver returns a ServiceResolver that routes
+// the kubernetes/default service with port 443 to loopback.
+func NewLoopbackServiceResolver(delegate ServiceResolver, host *url.URL) ServiceResolver {
+	return &loopbackResolver{
+		delegate: delegate,
+		host:     host,
+	}
+}
+
+type loopbackResolver struct {
+	delegate ServiceResolver
+	host     *url.URL
+}
+
+func (r *loopbackResolver) ResolveEndpoint(namespace, name string, port int32) (*url.URL, error) {
+	if namespace == "default" && name == "kubernetes" && port == 443 {
+		return r.host, nil
+	}
+	return r.delegate.ResolveEndpoint(namespace, name, port)
+}

vendor/k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheme
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/install"
+)
+
+var (
+	// Scheme defines methods for serializing and deserializing API objects.
+	Scheme = runtime.NewScheme()
+	// Codecs provides methods for retrieving codecs and serializers for specific
+	// versions and content types.
+	Codecs = serializer.NewCodecFactory(Scheme)
+)
+
+func init() {
+	install.Install(Scheme)
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/interface.go

@@ -0,0 +1,54 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package apiregistration
+
+import (
+	v1 "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
+	v1beta1 "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1beta1"
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+)
+
+// Interface provides access to each of this group's versions.
+type Interface interface {
+	// V1beta1 provides access to shared informers for resources in V1beta1.
+	V1beta1() v1beta1.Interface
+	// V1 provides access to shared informers for resources in V1.
+	V1() v1.Interface
+}
+
+type group struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// V1beta1 returns a new v1beta1.Interface.
+func (g *group) V1beta1() v1beta1.Interface {
+	return v1beta1.New(g.factory, g.namespace, g.tweakListOptions)
+}
+
+// V1 returns a new v1.Interface.
+func (g *group) V1() v1.Interface {
+	return v1.New(g.factory, g.namespace, g.tweakListOptions)
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1/apiservice.go

@@ -0,0 +1,89 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"context"
+	time "time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	clientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+	v1 "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+)
+
+// APIServiceInformer provides access to a shared informer and lister for
+// APIServices.
+type APIServiceInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1.APIServiceLister
+}
+
+type aPIServiceInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// NewAPIServiceInformer constructs a new informer for APIService type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewAPIServiceInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredAPIServiceInformer(client, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredAPIServiceInformer constructs a new informer for APIService type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredAPIServiceInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ApiregistrationV1().APIServices().List(context.TODO(), options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ApiregistrationV1().APIServices().Watch(context.TODO(), options)
+			},
+		},
+		&apiregistrationv1.APIService{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *aPIServiceInformer) defaultInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredAPIServiceInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *aPIServiceInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&apiregistrationv1.APIService{}, f.defaultInformer)
+}
+
+func (f *aPIServiceInformer) Lister() v1.APIServiceLister {
+	return v1.NewAPIServiceLister(f.Informer().GetIndexer())
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1/interface.go

@@ -0,0 +1,45 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+)
+
+// Interface provides access to all the informers in this group version.
+type Interface interface {
+	// APIServices returns a APIServiceInformer.
+	APIServices() APIServiceInformer
+}
+
+type version struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// APIServices returns a APIServiceInformer.
+func (v *version) APIServices() APIServiceInformer {
+	return &aPIServiceInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1beta1/apiservice.go

@@ -0,0 +1,89 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	"context"
+	time "time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+	clientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+	v1beta1 "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1beta1"
+)
+
+// APIServiceInformer provides access to a shared informer and lister for
+// APIServices.
+type APIServiceInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1beta1.APIServiceLister
+}
+
+type aPIServiceInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// NewAPIServiceInformer constructs a new informer for APIService type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewAPIServiceInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredAPIServiceInformer(client, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredAPIServiceInformer constructs a new informer for APIService type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredAPIServiceInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ApiregistrationV1beta1().APIServices().List(context.TODO(), options)
+			},
+			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.ApiregistrationV1beta1().APIServices().Watch(context.TODO(), options)
+			},
+		},
+		&apiregistrationv1beta1.APIService{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *aPIServiceInformer) defaultInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredAPIServiceInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *aPIServiceInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&apiregistrationv1beta1.APIService{}, f.defaultInformer)
+}
+
+func (f *aPIServiceInformer) Lister() v1beta1.APIServiceLister {
+	return v1beta1.NewAPIServiceLister(f.Informer().GetIndexer())
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1beta1/interface.go

@@ -0,0 +1,45 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+)
+
+// Interface provides access to all the informers in this group version.
+type Interface interface {
+	// APIServices returns a APIServiceInformer.
+	APIServices() APIServiceInformer
+}
+
+type version struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// APIServices returns a APIServiceInformer.
+func (v *version) APIServices() APIServiceInformer {
+	return &aPIServiceInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/factory.go

@@ -0,0 +1,251 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package externalversions
+
+import (
+	reflect "reflect"
+	sync "sync"
+	time "time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	cache "k8s.io/client-go/tools/cache"
+	clientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	apiregistration "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration"
+	internalinterfaces "k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces"
+)
+
+// SharedInformerOption defines the functional option type for SharedInformerFactory.
+type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory
+
+type sharedInformerFactory struct {
+	client           clientset.Interface
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+	lock             sync.Mutex
+	defaultResync    time.Duration
+	customResync     map[reflect.Type]time.Duration
+
+	informers map[reflect.Type]cache.SharedIndexInformer
+	// startedInformers is used for tracking which informers have been started.
+	// This allows Start() to be called multiple times safely.
+	startedInformers map[reflect.Type]bool
+	// wg tracks how many goroutines were started.
+	wg sync.WaitGroup
+	// shuttingDown is true when Shutdown has been called. It may still be running
+	// because it needs to wait for goroutines.
+	shuttingDown bool
+}
+
+// WithCustomResyncConfig sets a custom resync period for the specified informer types.
+func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		for k, v := range resyncConfig {
+			factory.customResync[reflect.TypeOf(k)] = v
+		}
+		return factory
+	}
+}
+
+// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory.
+func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		factory.tweakListOptions = tweakListOptions
+		return factory
+	}
+}
+
+// WithNamespace limits the SharedInformerFactory to the specified namespace.
+func WithNamespace(namespace string) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		factory.namespace = namespace
+		return factory
+	}
+}
+
+// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.
+func NewSharedInformerFactory(client clientset.Interface, defaultResync time.Duration) SharedInformerFactory {
+	return NewSharedInformerFactoryWithOptions(client, defaultResync)
+}
+
+// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
+// Listers obtained via this SharedInformerFactory will be subject to the same filters
+// as specified here.
+// Deprecated: Please use NewSharedInformerFactoryWithOptions instead
+func NewFilteredSharedInformerFactory(client clientset.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
+	return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions))
+}
+
+// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.
+func NewSharedInformerFactoryWithOptions(client clientset.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
+	factory := &sharedInformerFactory{
+		client:           client,
+		namespace:        v1.NamespaceAll,
+		defaultResync:    defaultResync,
+		informers:        make(map[reflect.Type]cache.SharedIndexInformer),
+		startedInformers: make(map[reflect.Type]bool),
+		customResync:     make(map[reflect.Type]time.Duration),
+	}
+
+	// Apply all options
+	for _, opt := range options {
+		factory = opt(factory)
+	}
+
+	return factory
+}
+
+func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if f.shuttingDown {
+		return
+	}
+
+	for informerType, informer := range f.informers {
+		if !f.startedInformers[informerType] {
+			f.wg.Add(1)
+			// We need a new variable in each loop iteration,
+			// otherwise the goroutine would use the loop variable
+			// and that keeps changing.
+			informer := informer
+			go func() {
+				defer f.wg.Done()
+				informer.Run(stopCh)
+			}()
+			f.startedInformers[informerType] = true
+		}
+	}
+}
+
+func (f *sharedInformerFactory) Shutdown() {
+	f.lock.Lock()
+	f.shuttingDown = true
+	f.lock.Unlock()
+
+	// Will return immediately if there is nothing to wait for.
+	f.wg.Wait()
+}
+
+func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
+	informers := func() map[reflect.Type]cache.SharedIndexInformer {
+		f.lock.Lock()
+		defer f.lock.Unlock()
+
+		informers := map[reflect.Type]cache.SharedIndexInformer{}
+		for informerType, informer := range f.informers {
+			if f.startedInformers[informerType] {
+				informers[informerType] = informer
+			}
+		}
+		return informers
+	}()
+
+	res := map[reflect.Type]bool{}
+	for informType, informer := range informers {
+		res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced)
+	}
+	return res
+}
+
+// InternalInformerFor returns the SharedIndexInformer for obj using an internal
+// client.
+func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	informerType := reflect.TypeOf(obj)
+	informer, exists := f.informers[informerType]
+	if exists {
+		return informer
+	}
+
+	resyncPeriod, exists := f.customResync[informerType]
+	if !exists {
+		resyncPeriod = f.defaultResync
+	}
+
+	informer = newFunc(f.client, resyncPeriod)
+	f.informers[informerType] = informer
+
+	return informer
+}
+
+// SharedInformerFactory provides shared informers for resources in all known
+// API group versions.
+//
+// It is typically used like this:
+//
+//	ctx, cancel := context.Background()
+//	defer cancel()
+//	factory := NewSharedInformerFactory(client, resyncPeriod)
+//	defer factory.WaitForStop()    // Returns immediately if nothing was started.
+//	genericInformer := factory.ForResource(resource)
+//	typedInformer := factory.SomeAPIGroup().V1().SomeType()
+//	factory.Start(ctx.Done())          // Start processing these informers.
+//	synced := factory.WaitForCacheSync(ctx.Done())
+//	for v, ok := range synced {
+//	    if !ok {
+//	        fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v)
+//	        return
+//	    }
+//	}
+//
+//	// Creating informers can also be created after Start, but then
+//	// Start must be called again:
+//	anotherGenericInformer := factory.ForResource(resource)
+//	factory.Start(ctx.Done())
+type SharedInformerFactory interface {
+	internalinterfaces.SharedInformerFactory
+
+	// Start initializes all requested informers. They are handled in goroutines
+	// which run until the stop channel gets closed.
+	Start(stopCh <-chan struct{})
+
+	// Shutdown marks a factory as shutting down. At that point no new
+	// informers can be started anymore and Start will return without
+	// doing anything.
+	//
+	// In addition, Shutdown blocks until all goroutines have terminated. For that
+	// to happen, the close channel(s) that they were started with must be closed,
+	// either before Shutdown gets called or while it is waiting.
+	//
+	// Shutdown may be called multiple times, even concurrently. All such calls will
+	// block until all goroutines have terminated.
+	Shutdown()
+
+	// WaitForCacheSync blocks until all started informers' caches were synced
+	// or the stop channel gets closed.
+	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
+
+	// ForResource gives generic access to a shared informer of the matching type.
+	ForResource(resource schema.GroupVersionResource) (GenericInformer, error)
+
+	// InternalInformerFor returns the SharedIndexInformer for obj using an internal
+	// client.
+	InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer
+
+	Apiregistration() apiregistration.Interface
+}
+
+func (f *sharedInformerFactory) Apiregistration() apiregistration.Interface {
+	return apiregistration.New(f, f.namespace, f.tweakListOptions)
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/generic.go

@@ -0,0 +1,67 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package externalversions
+
+import (
+	"fmt"
+
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	cache "k8s.io/client-go/tools/cache"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	v1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+)
+
+// GenericInformer is type of SharedIndexInformer which will locate and delegate to other
+// sharedInformers based on type
+type GenericInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() cache.GenericLister
+}
+
+type genericInformer struct {
+	informer cache.SharedIndexInformer
+	resource schema.GroupResource
+}
+
+// Informer returns the SharedIndexInformer.
+func (f *genericInformer) Informer() cache.SharedIndexInformer {
+	return f.informer
+}
+
+// Lister returns the GenericLister.
+func (f *genericInformer) Lister() cache.GenericLister {
+	return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource)
+}
+
+// ForResource gives generic access to a shared informer of the matching type
+// TODO extend this to unknown resources with a client pool
+func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
+	switch resource {
+	// Group=apiregistration.k8s.io, Version=v1
+	case v1.SchemeGroupVersion.WithResource("apiservices"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Apiregistration().V1().APIServices().Informer()}, nil
+
+		// Group=apiregistration.k8s.io, Version=v1beta1
+	case v1beta1.SchemeGroupVersion.WithResource("apiservices"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Apiregistration().V1beta1().APIServices().Informer()}, nil
+
+	}
+
+	return nil, fmt.Errorf("no informer found for %v", resource)
+}

vendor/k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go

@@ -0,0 +1,40 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package internalinterfaces
+
+import (
+	time "time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	cache "k8s.io/client-go/tools/cache"
+	clientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+)
+
+// NewInformerFunc takes clientset.Interface and time.Duration to return a SharedIndexInformer.
+type NewInformerFunc func(clientset.Interface, time.Duration) cache.SharedIndexInformer
+
+// SharedInformerFactory a small interface to allow for adding an informer without an import cycle
+type SharedInformerFactory interface {
+	Start(stopCh <-chan struct{})
+	InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer
+}
+
+// TweakListOptionsFunc is a function that transforms a v1.ListOptions.
+type TweakListOptionsFunc func(*v1.ListOptions)

vendor/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1/apiservice.go

@@ -0,0 +1,68 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+)
+
+// APIServiceLister helps list APIServices.
+// All objects returned here must be treated as read-only.
+type APIServiceLister interface {
+	// List lists all APIServices in the indexer.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*v1.APIService, err error)
+	// Get retrieves the APIService from the index for a given name.
+	// Objects returned here must be treated as read-only.
+	Get(name string) (*v1.APIService, error)
+	APIServiceListerExpansion
+}
+
+// aPIServiceLister implements the APIServiceLister interface.
+type aPIServiceLister struct {
+	indexer cache.Indexer
+}
+
+// NewAPIServiceLister returns a new APIServiceLister.
+func NewAPIServiceLister(indexer cache.Indexer) APIServiceLister {
+	return &aPIServiceLister{indexer: indexer}
+}
+
+// List lists all APIServices in the indexer.
+func (s *aPIServiceLister) List(selector labels.Selector) (ret []*v1.APIService, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1.APIService))
+	})
+	return ret, err
+}
+
+// Get retrieves the APIService from the index for a given name.
+func (s *aPIServiceLister) Get(name string) (*v1.APIService, error) {
+	obj, exists, err := s.indexer.GetByKey(name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1.Resource("apiservice"), name)
+	}
+	return obj.(*v1.APIService), nil
+}

vendor/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1/expansion_generated.go

@@ -0,0 +1,23 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+// APIServiceListerExpansion allows custom methods to be added to
+// APIServiceLister.
+type APIServiceListerExpansion interface{}

vendor/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1beta1/apiservice.go

@@ -0,0 +1,68 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+	v1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+)
+
+// APIServiceLister helps list APIServices.
+// All objects returned here must be treated as read-only.
+type APIServiceLister interface {
+	// List lists all APIServices in the indexer.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*v1beta1.APIService, err error)
+	// Get retrieves the APIService from the index for a given name.
+	// Objects returned here must be treated as read-only.
+	Get(name string) (*v1beta1.APIService, error)
+	APIServiceListerExpansion
+}
+
+// aPIServiceLister implements the APIServiceLister interface.
+type aPIServiceLister struct {
+	indexer cache.Indexer
+}
+
+// NewAPIServiceLister returns a new APIServiceLister.
+func NewAPIServiceLister(indexer cache.Indexer) APIServiceLister {
+	return &aPIServiceLister{indexer: indexer}
+}
+
+// List lists all APIServices in the indexer.
+func (s *aPIServiceLister) List(selector labels.Selector) (ret []*v1beta1.APIService, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.APIService))
+	})
+	return ret, err
+}
+
+// Get retrieves the APIService from the index for a given name.
+func (s *aPIServiceLister) Get(name string) (*v1beta1.APIService, error) {
+	obj, exists, err := s.indexer.GetByKey(name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1beta1.Resource("apiservice"), name)
+	}
+	return obj.(*v1beta1.APIService), nil
+}

vendor/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1beta1/expansion_generated.go

@@ -0,0 +1,23 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1beta1
+
+// APIServiceListerExpansion allows custom methods to be added to
+// APIServiceLister.
+type APIServiceListerExpansion interface{}

vendor/k8s.io/kube-aggregator/pkg/controllers/cache.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+// WaitForCacheSync is a wrapper around cache.WaitForCacheSync that generates log messages
+// indicating that the controller identified by controllerName is waiting for syncs, followed by
+// either a successful or failed sync.
+func WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs ...cache.InformerSynced) bool {
+	klog.Infof("Waiting for caches to sync for %s controller", controllerName)
+
+	if !cache.WaitForCacheSync(stopCh, cacheSyncs...) {
+		utilruntime.HandleError(fmt.Errorf("Unable to sync caches for %s controller", controllerName))
+		return false
+	}
+
+	klog.Infof("Caches are synced for %s controller", controllerName)
+	return true
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go

@@ -0,0 +1,351 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	restful "github.com/emicklei/go-restful/v3"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/server"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-openapi/pkg/aggregator"
+	"k8s.io/kube-openapi/pkg/builder"
+	"k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/handler"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+// SpecAggregator calls out to http handlers of APIServices and merges specs. It keeps state of the last
+// known specs including the http etag.
+type SpecAggregator interface {
+	AddUpdateAPIService(handler http.Handler, apiService *v1.APIService) error
+	UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
+	RemoveAPIServiceSpec(apiServiceName string) error
+	GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
+	GetAPIServiceNames() []string
+}
+
+const (
+	aggregatorUser                = "system:aggregator"
+	specDownloadTimeout           = 60 * time.Second
+	localDelegateChainNamePrefix  = "k8s_internal_local_delegation_chain_"
+	localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"
+
+	// A randomly generated UUID to differentiate local and remote eTags.
+	locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-"
+)
+
+// IsLocalAPIService returns true for local specs from delegates.
+func IsLocalAPIService(apiServiceName string) bool {
+	return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
+}
+
+// GetAPIServiceNames returns the names of APIServices recorded in specAggregator.openAPISpecs.
+// We use this function to pass the names of local APIServices to the controller in this package,
+// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
+func (s *specAggregator) GetAPIServiceNames() []string {
+	names := make([]string, 0, len(s.openAPISpecs))
+	for key := range s.openAPISpecs {
+		names = append(names, key)
+	}
+	return names
+}
+
+// BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
+func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
+	config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) {
+	s := &specAggregator{
+		openAPISpecs: map[string]*openAPISpecInfo{},
+	}
+
+	i := 0
+	// Build Aggregator's spec
+	aggregatorOpenAPISpec, err := builder.BuildOpenAPISpec(webServices, config)
+	if err != nil {
+		return nil, err
+	}
+
+	// Reserving non-name spec for aggregator's Spec.
+	s.addLocalSpec(aggregatorOpenAPISpec, nil, fmt.Sprintf(localDelegateChainNamePattern, i), "")
+	i++
+	for delegate := delegationTarget; delegate != nil; delegate = delegate.NextDelegate() {
+		handler := delegate.UnprotectedHandler()
+		if handler == nil {
+			continue
+		}
+		delegateSpec, etag, _, err := downloader.Download(handler, "")
+		if err != nil {
+			// ignore errors for the empty delegate we attach at the end the chain
+			// atm the empty delegate returns 503 when the server hasn't been fully initialized
+			// and the spec downloader only silences 404s
+			if len(delegate.ListedPaths()) == 0 && delegate.NextDelegate() == nil {
+				continue
+			}
+			return nil, err
+		}
+		if delegateSpec == nil {
+			continue
+		}
+		s.addLocalSpec(delegateSpec, handler, fmt.Sprintf(localDelegateChainNamePattern, i), etag)
+		i++
+	}
+
+	// Build initial spec to serve.
+	klog.V(2).Infof("Building initial OpenAPI spec")
+	defer func(start time.Time) {
+		duration := time.Since(start)
+		klog.V(2).Infof("Finished initial OpenAPI spec generation after %v", duration)
+
+		regenerationCounter.With(map[string]string{"apiservice": "*", "reason": "startup"})
+		regenerationDurationGauge.With(map[string]string{"reason": "startup"}).Set(duration.Seconds())
+	}(time.Now())
+	specToServe, err := s.buildOpenAPISpec()
+	if err != nil {
+		return nil, err
+	}
+
+	// Install handler
+	s.openAPIVersionedService, err = handler.NewOpenAPIService(specToServe)
+	if err != nil {
+		return nil, err
+	}
+	err = s.openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", pathHandler)
+	if err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+type specAggregator struct {
+	// mutex protects all members of this struct.
+	rwMutex sync.RWMutex
+
+	// Map of API Services' OpenAPI specs by their name
+	openAPISpecs map[string]*openAPISpecInfo
+
+	// provided for dynamic OpenAPI spec
+	openAPIVersionedService *handler.OpenAPIService
+}
+
+var _ SpecAggregator = &specAggregator{}
+
+// This function is not thread safe as it only being called on startup.
+func (s *specAggregator) addLocalSpec(spec *spec.Swagger, localHandler http.Handler, name, etag string) {
+	localAPIService := v1.APIService{}
+	localAPIService.Name = name
+	s.openAPISpecs[name] = &openAPISpecInfo{
+		etag:       etag,
+		apiService: localAPIService,
+		handler:    localHandler,
+		spec:       spec,
+	}
+}
+
+// openAPISpecInfo is used to store OpenAPI spec with its priority.
+// It can be used to sort specs with their priorities.
+type openAPISpecInfo struct {
+	apiService v1.APIService
+
+	// Specification of this API Service. If null then the spec is not loaded yet.
+	spec    *spec.Swagger
+	handler http.Handler
+	etag    string
+}
+
+// buildOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
+func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err error) {
+	specs := []openAPISpecInfo{}
+	for _, specInfo := range s.openAPISpecs {
+		if specInfo.spec == nil {
+			continue
+		}
+		// Copy the spec before removing the defaults.
+		localSpec := *specInfo.spec
+		localSpecInfo := *specInfo
+		localSpecInfo.spec = &localSpec
+		localSpecInfo.spec.Definitions = handler.PruneDefaults(specInfo.spec.Definitions)
+		specs = append(specs, localSpecInfo)
+	}
+	if len(specs) == 0 {
+		return &spec.Swagger{}, nil
+	}
+	sortByPriority(specs)
+	for _, specInfo := range specs {
+		if specToReturn == nil {
+			specToReturn = &spec.Swagger{}
+			*specToReturn = *specInfo.spec
+			// Paths and Definitions are set by MergeSpecsIgnorePathConflict
+			specToReturn.Paths = nil
+			specToReturn.Definitions = nil
+		}
+		if err := aggregator.MergeSpecsIgnorePathConflict(specToReturn, specInfo.spec); err != nil {
+			return nil, err
+		}
+	}
+
+	return specToReturn, nil
+}
+
+// updateOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
+func (s *specAggregator) updateOpenAPISpec() error {
+	if s.openAPIVersionedService == nil {
+		return nil
+	}
+	specToServe, err := s.buildOpenAPISpec()
+	if err != nil {
+		return err
+	}
+	return s.openAPIVersionedService.UpdateSpec(specToServe)
+}
+
+// tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
+// if the update fails.
+func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
+	if specInfo == nil {
+		return fmt.Errorf("invalid input: specInfo must be non-nil")
+	}
+	_, updated := s.openAPISpecs[specInfo.apiService.Name]
+	origSpecInfo, existedBefore := s.openAPISpecs[specInfo.apiService.Name]
+	s.openAPISpecs[specInfo.apiService.Name] = specInfo
+
+	// Skip aggregation if OpenAPI spec didn't change
+	if existedBefore && origSpecInfo != nil && origSpecInfo.etag == specInfo.etag {
+		return nil
+	}
+	klog.V(2).Infof("Updating OpenAPI spec because %s is updated", specInfo.apiService.Name)
+	defer func(start time.Time) {
+		duration := time.Since(start)
+		klog.V(2).Infof("Finished OpenAPI spec generation after %v", duration)
+
+		reason := "add"
+		if updated {
+			reason = "update"
+		}
+
+		regenerationCounter.With(map[string]string{"apiservice": specInfo.apiService.Name, "reason": reason})
+		regenerationDurationGauge.With(map[string]string{"reason": reason}).Set(duration.Seconds())
+	}(time.Now())
+	if err := s.updateOpenAPISpec(); err != nil {
+		if existedBefore {
+			s.openAPISpecs[specInfo.apiService.Name] = origSpecInfo
+		} else {
+			delete(s.openAPISpecs, specInfo.apiService.Name)
+		}
+		return err
+	}
+	return nil
+}
+
+// tryDeleteServiceSpecs tries delete specified specInfo from openAPISpecs map, and keeps the map intact
+// if the update fails.
+func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error {
+	orgSpecInfo, exists := s.openAPISpecs[apiServiceName]
+	if !exists {
+		return nil
+	}
+	delete(s.openAPISpecs, apiServiceName)
+	klog.V(2).Infof("Updating OpenAPI spec because %s is removed", apiServiceName)
+	defer func(start time.Time) {
+		duration := time.Since(start)
+		klog.V(2).Infof("Finished OpenAPI spec generation after %v", duration)
+
+		regenerationCounter.With(map[string]string{"apiservice": apiServiceName, "reason": "delete"})
+		regenerationDurationGauge.With(map[string]string{"reason": "delete"}).Set(duration.Seconds())
+	}(time.Now())
+	if err := s.updateOpenAPISpec(); err != nil {
+		s.openAPISpecs[apiServiceName] = orgSpecInfo
+		return err
+	}
+	return nil
+}
+
+// UpdateAPIServiceSpec updates the api service's OpenAPI spec. It is thread safe.
+func (s *specAggregator) UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+
+	specInfo, existingService := s.openAPISpecs[apiServiceName]
+	if !existingService {
+		return fmt.Errorf("APIService %q does not exists", apiServiceName)
+	}
+
+	// For APIServices (non-local) specs, only merge their /apis/ prefixed endpoint as it is the only paths
+	// proxy handler delegates.
+	if specInfo.apiService.Spec.Service != nil {
+		spec = aggregator.FilterSpecByPathsWithoutSideEffects(spec, []string{"/apis/"})
+	}
+
+	return s.tryUpdatingServiceSpecs(&openAPISpecInfo{
+		apiService: specInfo.apiService,
+		spec:       spec,
+		handler:    specInfo.handler,
+		etag:       etag,
+	})
+}
+
+// AddUpdateAPIService adds or updates the api service. It is thread safe.
+func (s *specAggregator) AddUpdateAPIService(handler http.Handler, apiService *v1.APIService) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+
+	if apiService.Spec.Service == nil {
+		// All local specs should be already aggregated using local delegate chain
+		return nil
+	}
+
+	newSpec := &openAPISpecInfo{
+		apiService: *apiService,
+		handler:    handler,
+	}
+	if specInfo, existingService := s.openAPISpecs[apiService.Name]; existingService {
+		newSpec.etag = specInfo.etag
+		newSpec.spec = specInfo.spec
+	}
+	return s.tryUpdatingServiceSpecs(newSpec)
+}
+
+// RemoveAPIServiceSpec removes an api service from OpenAPI aggregation. If it does not exist, no error is returned.
+// It is thread safe.
+func (s *specAggregator) RemoveAPIServiceSpec(apiServiceName string) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+
+	if _, existingService := s.openAPISpecs[apiServiceName]; !existingService {
+		return nil
+	}
+
+	return s.tryDeleteServiceSpecs(apiServiceName)
+}
+
+// GetAPIServiceSpec returns api service spec info
+func (s *specAggregator) GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool) {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	if info, existingService := s.openAPISpecs[apiServiceName]; existingService {
+		return info.handler, info.etag, true
+	}
+	return nil, "", false
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+// Downloader is the OpenAPI downloader type. It will try to download spec from /openapi/v2 or /swagger.json endpoint.
+type Downloader struct {
+}
+
+// NewDownloader creates a new OpenAPI Downloader.
+func NewDownloader() Downloader {
+	return Downloader{}
+}
+
+func (s *Downloader) handlerWithUser(handler http.Handler, info user.Info) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		req = req.WithContext(request.WithUser(req.Context(), info))
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func etagFor(data []byte) string {
+	return fmt.Sprintf("%s%X\"", locallyGeneratedEtagPrefix, sha512.Sum512(data))
+}
+
+// Download downloads openAPI spec from /openapi/v2 endpoint of the given handler.
+// httpStatus is only valid if err == nil
+func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *spec.Swagger, newEtag string, httpStatus int, err error) {
+	handler = s.handlerWithUser(handler, &user.DefaultInfo{Name: aggregatorUser})
+	handler = http.TimeoutHandler(handler, specDownloadTimeout, "request timed out")
+
+	req, err := http.NewRequest("GET", "/openapi/v2", nil)
+	if err != nil {
+		return nil, "", 0, err
+	}
+	req.Header.Add("Accept", "application/json")
+
+	// Only pass eTag if it is not generated locally
+	if len(etag) > 0 && !strings.HasPrefix(etag, locallyGeneratedEtagPrefix) {
+		req.Header.Add("If-None-Match", etag)
+	}
+
+	writer := newInMemoryResponseWriter()
+	handler.ServeHTTP(writer, req)
+
+	switch writer.respCode {
+	case http.StatusNotModified:
+		if len(etag) == 0 {
+			return nil, etag, http.StatusNotModified, fmt.Errorf("http.StatusNotModified is not allowed in absence of etag")
+		}
+		return nil, etag, http.StatusNotModified, nil
+	case http.StatusNotFound:
+		// Gracefully skip 404, assuming the server won't provide any spec
+		return nil, "", http.StatusNotFound, nil
+	case http.StatusOK:
+		openAPISpec := &spec.Swagger{}
+		if err := openAPISpec.UnmarshalJSON(writer.data); err != nil {
+			return nil, "", 0, err
+		}
+		newEtag = writer.Header().Get("Etag")
+		if len(newEtag) == 0 {
+			newEtag = etagFor(writer.data)
+			if len(etag) > 0 && strings.HasPrefix(etag, locallyGeneratedEtagPrefix) {
+				// The function call with an etag and server does not report an etag.
+				// That means this server does not support etag and the etag that passed
+				// to the function generated previously by us. Just compare etags and
+				// return StatusNotModified if they are the same.
+				if etag == newEtag {
+					return nil, etag, http.StatusNotModified, nil
+				}
+			}
+		}
+		return openAPISpec, newEtag, http.StatusOK, nil
+	default:
+		return nil, "", 0, fmt.Errorf("failed to retrieve openAPI spec, http error: %s", writer.String())
+	}
+}
+
+// inMemoryResponseWriter is a http.Writer that keep the response in memory.
+type inMemoryResponseWriter struct {
+	writeHeaderCalled bool
+	header            http.Header
+	respCode          int
+	data              []byte
+}
+
+func newInMemoryResponseWriter() *inMemoryResponseWriter {
+	return &inMemoryResponseWriter{header: http.Header{}}
+}
+
+func (r *inMemoryResponseWriter) Header() http.Header {
+	return r.header
+}
+
+func (r *inMemoryResponseWriter) WriteHeader(code int) {
+	r.writeHeaderCalled = true
+	r.respCode = code
+}
+
+func (r *inMemoryResponseWriter) Write(in []byte) (int, error) {
+	if !r.writeHeaderCalled {
+		r.WriteHeader(http.StatusOK)
+	}
+	r.data = append(r.data, in...)
+	return len(in), nil
+}
+
+func (r *inMemoryResponseWriter) String() string {
+	s := fmt.Sprintf("ResponseCode: %d", r.respCode)
+	if r.data != nil {
+		s += fmt.Sprintf(", Body: %s", string(r.data))
+	}
+	if r.header != nil {
+		s += fmt.Sprintf(", Header: %s", r.header)
+	}
+	return s
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/metrics.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+var (
+	regenerationCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "aggregator_openapi_v2_regeneration_count",
+			Help:           "Counter of OpenAPI v2 spec regeneration count broken down by causing APIService name and reason.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"apiservice", "reason"},
+	)
+	regenerationDurationGauge = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Name:           "aggregator_openapi_v2_regeneration_duration",
+			Help:           "Gauge of OpenAPI v2 spec regeneration duration in seconds.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"reason"},
+	)
+)
+
+func init() {
+	legacyregistry.MustRegister(regenerationCounter)
+	legacyregistry.MustRegister(regenerationDurationGauge)
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/priority.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"sort"
+)
+
+// byPriority can be used in sort.Sort to sort specs with their priorities.
+type byPriority struct {
+	specs           []openAPISpecInfo
+	groupPriorities map[string]int32
+}
+
+func (a byPriority) Len() int      { return len(a.specs) }
+func (a byPriority) Swap(i, j int) { a.specs[i], a.specs[j] = a.specs[j], a.specs[i] }
+func (a byPriority) Less(i, j int) bool {
+	// All local specs will come first
+	if a.specs[i].apiService.Spec.Service == nil && a.specs[j].apiService.Spec.Service != nil {
+		return true
+	}
+	if a.specs[i].apiService.Spec.Service != nil && a.specs[j].apiService.Spec.Service == nil {
+		return false
+	}
+	// WARNING: This will result in not following priorities for local APIServices.
+	if a.specs[i].apiService.Spec.Service == nil {
+		// Sort local specs with their name. This is the order in the delegation chain (aggregator first).
+		return a.specs[i].apiService.Name < a.specs[j].apiService.Name
+	}
+	var iPriority, jPriority int32
+	if a.specs[i].apiService.Spec.Group == a.specs[j].apiService.Spec.Group {
+		iPriority = a.specs[i].apiService.Spec.VersionPriority
+		jPriority = a.specs[i].apiService.Spec.VersionPriority
+	} else {
+		iPriority = a.groupPriorities[a.specs[i].apiService.Spec.Group]
+		jPriority = a.groupPriorities[a.specs[j].apiService.Spec.Group]
+	}
+	if iPriority != jPriority {
+		// Sort by priority, higher first
+		return iPriority > jPriority
+	}
+	// Sort by service name.
+	return a.specs[i].apiService.Name < a.specs[j].apiService.Name
+}
+
+func sortByPriority(specs []openAPISpecInfo) {
+	b := byPriority{
+		specs:           specs,
+		groupPriorities: map[string]int32{},
+	}
+	for _, spec := range specs {
+		if spec.apiService.Spec.Service == nil {
+			continue
+		}
+		if pr, found := b.groupPriorities[spec.apiService.Spec.Group]; !found || spec.apiService.Spec.GroupPriorityMinimum > pr {
+			b.groupPriorities[spec.apiService.Spec.Group] = spec.apiService.Spec.GroupPriorityMinimum
+		}
+	}
+	sort.Sort(b)
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator"
+)
+
+const (
+	successfulUpdateDelay      = time.Minute
+	successfulUpdateDelayLocal = time.Second
+	failedUpdateMaxExpDelay    = time.Hour
+)
+
+type syncAction int
+
+const (
+	syncRequeue syncAction = iota
+	syncRequeueRateLimited
+	syncNothing
+)
+
+// AggregationController periodically check for changes in OpenAPI specs of APIServices and update/remove
+// them if necessary.
+type AggregationController struct {
+	openAPIAggregationManager aggregator.SpecAggregator
+	queue                     workqueue.RateLimitingInterface
+	downloader                *aggregator.Downloader
+
+	// To allow injection for testing.
+	syncHandler func(key string) (syncAction, error)
+}
+
+// NewAggregationController creates new OpenAPI aggregation controller.
+func NewAggregationController(downloader *aggregator.Downloader, openAPIAggregationManager aggregator.SpecAggregator) *AggregationController {
+	c := &AggregationController{
+		openAPIAggregationManager: openAPIAggregationManager,
+		queue: workqueue.NewNamedRateLimitingQueue(
+			workqueue.NewItemExponentialFailureRateLimiter(successfulUpdateDelay, failedUpdateMaxExpDelay),
+			"open_api_aggregation_controller",
+		),
+		downloader: downloader,
+	}
+
+	c.syncHandler = c.sync
+
+	// update each service at least once, also those which are not coming from APIServices, namely local services
+	for _, name := range openAPIAggregationManager.GetAPIServiceNames() {
+		c.queue.AddAfter(name, time.Second)
+	}
+
+	return c
+}
+
+// Run starts OpenAPI AggregationController
+func (c *AggregationController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting OpenAPI AggregationController")
+	defer klog.Info("Shutting down OpenAPI AggregationController")
+
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+func (c *AggregationController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *AggregationController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	defer c.queue.Done(key)
+	if quit {
+		return false
+	}
+
+	if aggregator.IsLocalAPIService(key.(string)) {
+		// for local delegation targets that are aggregated once per second, log at
+		// higher level to avoid flooding the log
+		klog.V(6).Infof("OpenAPI AggregationController: Processing item %s", key)
+	} else {
+		klog.V(4).Infof("OpenAPI AggregationController: Processing item %s", key)
+	}
+
+	action, err := c.syncHandler(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+	} else {
+		utilruntime.HandleError(fmt.Errorf("loading OpenAPI spec for %q failed with: %v", key, err))
+	}
+
+	switch action {
+	case syncRequeue:
+		if aggregator.IsLocalAPIService(key.(string)) {
+			klog.V(7).Infof("OpenAPI AggregationController: action for local item %s: Requeue after %s.", key, successfulUpdateDelayLocal)
+			c.queue.AddAfter(key, successfulUpdateDelayLocal)
+		} else {
+			klog.V(7).Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
+			c.queue.AddAfter(key, successfulUpdateDelay)
+		}
+	case syncRequeueRateLimited:
+		klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key)
+		c.queue.AddRateLimited(key)
+	case syncNothing:
+		klog.Infof("OpenAPI AggregationController: action for item %s: Nothing (removed from the queue).", key)
+	}
+
+	return true
+}
+
+func (c *AggregationController) sync(key string) (syncAction, error) {
+	handler, etag, exists := c.openAPIAggregationManager.GetAPIServiceInfo(key)
+	if !exists || handler == nil {
+		return syncNothing, nil
+	}
+	returnSpec, newEtag, httpStatus, err := c.downloader.Download(handler, etag)
+	switch {
+	case err != nil:
+		return syncRequeueRateLimited, err
+	case httpStatus == http.StatusNotModified:
+	case httpStatus == http.StatusNotFound || returnSpec == nil:
+		return syncRequeueRateLimited, fmt.Errorf("OpenAPI spec does not exist")
+	case httpStatus == http.StatusOK:
+		if err := c.openAPIAggregationManager.UpdateAPIServiceSpec(key, returnSpec, newEtag); err != nil {
+			return syncRequeueRateLimited, err
+		}
+	}
+	return syncRequeue, nil
+}
+
+// AddAPIService adds a new API Service to OpenAPI Aggregation.
+func (c *AggregationController) AddAPIService(handler http.Handler, apiService *v1.APIService) {
+	if apiService.Spec.Service == nil {
+		return
+	}
+	if err := c.openAPIAggregationManager.AddUpdateAPIService(handler, apiService); err != nil {
+		utilruntime.HandleError(fmt.Errorf("adding %q to AggregationController failed with: %v", apiService.Name, err))
+	}
+	c.queue.AddAfter(apiService.Name, time.Second)
+}
+
+// UpdateAPIService updates API Service's info and handler.
+func (c *AggregationController) UpdateAPIService(handler http.Handler, apiService *v1.APIService) {
+	if apiService.Spec.Service == nil {
+		return
+	}
+	if err := c.openAPIAggregationManager.AddUpdateAPIService(handler, apiService); err != nil {
+		utilruntime.HandleError(fmt.Errorf("updating %q to AggregationController failed with: %v", apiService.Name, err))
+	}
+	key := apiService.Name
+	if c.queue.NumRequeues(key) > 0 {
+		// The item has failed before. Remove it from failure queue and
+		// update it in a second
+		c.queue.Forget(key)
+		c.queue.AddAfter(key, time.Second)
+	}
+	// Else: The item has been succeeded before and it will be updated soon (after successfulUpdateDelay)
+	// we don't add it again as it will cause a duplication of items.
+}
+
+// RemoveAPIService removes API Service from OpenAPI Aggregation Controller.
+func (c *AggregationController) RemoveAPIService(apiServiceName string) {
+	if err := c.openAPIAggregationManager.RemoveAPIServiceSpec(apiServiceName); err != nil {
+		utilruntime.HandleError(fmt.Errorf("removing %q from AggregationController failed with: %v", apiServiceName, err))
+	}
+	// This will only remove it if it was failing before. If it was successful, processNextWorkItem will figure it out
+	// and will not add it again to the queue.
+	c.queue.Forget(apiServiceName)
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/aggregator.go

@@ -0,0 +1,276 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/mux"
+	"k8s.io/klog/v2"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/handler3"
+	"k8s.io/kube-openapi/pkg/openapiconv"
+
+	v2aggregator "k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator"
+)
+
+// SpecProxier proxies OpenAPI V3 requests to their respective APIService
+type SpecProxier interface {
+	AddUpdateAPIService(handler http.Handler, apiService *v1.APIService)
+	UpdateAPIServiceSpec(apiServiceName string) error
+	RemoveAPIServiceSpec(apiServiceName string)
+	GetAPIServiceNames() []string
+}
+
+const (
+	aggregatorUser                = "system:aggregator"
+	specDownloadTimeout           = 60 * time.Second
+	localDelegateChainNamePrefix  = "k8s_internal_local_delegation_chain_"
+	localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"
+	openAPIV2Converter            = "openapiv2converter"
+)
+
+// IsLocalAPIService returns true for local specs from delegates.
+func IsLocalAPIService(apiServiceName string) bool {
+	return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
+}
+
+// GetAPIServiceNames returns the names of APIServices recorded in apiServiceInfo.
+// We use this function to pass the names of local APIServices to the controller in this package,
+// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
+func (s *specProxier) GetAPIServiceNames() []string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	names := make([]string, 0, len(s.apiServiceInfo))
+	for key := range s.apiServiceInfo {
+		names = append(names, key)
+	}
+	return names
+}
+
+// BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
+func BuildAndRegisterAggregator(downloader Downloader, delegationTarget server.DelegationTarget, pathHandler common.PathHandlerByGroupVersion) (SpecProxier, error) {
+	s := &specProxier{
+		apiServiceInfo: map[string]*openAPIV3APIServiceInfo{},
+		downloader:     downloader,
+	}
+
+	i := 1
+	for delegate := delegationTarget; delegate != nil; delegate = delegate.NextDelegate() {
+		handler := delegate.UnprotectedHandler()
+		if handler == nil {
+			continue
+		}
+
+		apiServiceName := fmt.Sprintf(localDelegateChainNamePattern, i)
+		localAPIService := v1.APIService{}
+		localAPIService.Name = apiServiceName
+		s.AddUpdateAPIService(handler, &localAPIService)
+		s.UpdateAPIServiceSpec(apiServiceName)
+		i++
+	}
+
+	handler, err := handler3.NewOpenAPIService(nil)
+	if err != nil {
+		return s, err
+	}
+	s.openAPIV2ConverterHandler = handler
+	openAPIV2ConverterMux := mux.NewPathRecorderMux(openAPIV2Converter)
+	s.openAPIV2ConverterHandler.RegisterOpenAPIV3VersionedService("/openapi/v3", openAPIV2ConverterMux)
+	openAPIV2ConverterAPIService := v1.APIService{}
+	openAPIV2ConverterAPIService.Name = openAPIV2Converter
+	s.AddUpdateAPIService(openAPIV2ConverterMux, &openAPIV2ConverterAPIService)
+	s.register(pathHandler)
+
+	return s, nil
+}
+
+// AddUpdateAPIService adds or updates the api service. It is thread safe.
+func (s *specProxier) AddUpdateAPIService(handler http.Handler, apiservice *v1.APIService) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	// If the APIService is being updated, use the existing struct.
+	if apiServiceInfo, ok := s.apiServiceInfo[apiservice.Name]; ok {
+		apiServiceInfo.apiService = *apiservice
+		apiServiceInfo.handler = handler
+	}
+	s.apiServiceInfo[apiservice.Name] = &openAPIV3APIServiceInfo{
+		apiService: *apiservice,
+		handler:    handler,
+	}
+}
+
+func getGroupVersionStringFromAPIService(apiService v1.APIService) string {
+	if apiService.Spec.Group == "" && apiService.Spec.Version == "" {
+		return ""
+	}
+	return "apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
+}
+
+// UpdateAPIServiceSpec updates all the OpenAPI v3 specs that the APIService serves.
+// It is thread safe.
+func (s *specProxier) UpdateAPIServiceSpec(apiServiceName string) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	return s.updateAPIServiceSpecLocked(apiServiceName)
+}
+
+func (s *specProxier) updateAPIServiceSpecLocked(apiServiceName string) error {
+	apiService, exists := s.apiServiceInfo[apiServiceName]
+	if !exists {
+		return fmt.Errorf("APIService %s does not exist for update", apiServiceName)
+	}
+
+	if !apiService.isLegacyAPIService {
+		gv, httpStatus, err := s.downloader.OpenAPIV3Root(apiService.handler)
+		if err != nil {
+			return err
+		}
+		if httpStatus == http.StatusNotFound {
+			apiService.isLegacyAPIService = true
+		} else {
+			s.apiServiceInfo[apiServiceName].discovery = gv
+			return nil
+		}
+	}
+
+	newDownloader := v2aggregator.Downloader{}
+	v2Spec, etag, httpStatus, err := newDownloader.Download(apiService.handler, apiService.etag)
+	if err != nil {
+		return err
+	}
+	apiService.etag = etag
+	if httpStatus == http.StatusOK {
+		v3Spec := openapiconv.ConvertV2ToV3(v2Spec)
+		s.openAPIV2ConverterHandler.UpdateGroupVersion(getGroupVersionStringFromAPIService(apiService.apiService), v3Spec)
+		s.updateAPIServiceSpecLocked(openAPIV2Converter)
+	}
+	return nil
+}
+
+type specProxier struct {
+	// mutex protects all members of this struct.
+	rwMutex sync.RWMutex
+
+	// OpenAPI V3 specs by APIService name
+	apiServiceInfo map[string]*openAPIV3APIServiceInfo
+
+	// For downloading the OpenAPI v3 specs from apiservices
+	downloader Downloader
+
+	openAPIV2ConverterHandler *handler3.OpenAPIService
+}
+
+var _ SpecProxier = &specProxier{}
+
+type openAPIV3APIServiceInfo struct {
+	apiService v1.APIService
+	handler    http.Handler
+	discovery  *handler3.OpenAPIV3Discovery
+
+	// These fields are only used if the /openapi/v3 endpoint is not served by an APIService
+	// Legacy APIService indicates that an APIService does not support OpenAPI V3, and the OpenAPI V2
+	// will be downloaded, converted to V3 (lossy), and served by the aggregator
+	etag               string
+	isLegacyAPIService bool
+}
+
+// RemoveAPIServiceSpec removes an api service from the OpenAPI map. If it does not exist, no error is returned.
+// It is thread safe.
+func (s *specProxier) RemoveAPIServiceSpec(apiServiceName string) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	if apiServiceInfo, ok := s.apiServiceInfo[apiServiceName]; ok {
+		s.openAPIV2ConverterHandler.DeleteGroupVersion(getGroupVersionStringFromAPIService(apiServiceInfo.apiService))
+		delete(s.apiServiceInfo, apiServiceName)
+	}
+}
+
+func (s *specProxier) getOpenAPIV3Root() handler3.OpenAPIV3Discovery {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	merged := handler3.OpenAPIV3Discovery{
+		Paths: make(map[string]handler3.OpenAPIV3DiscoveryGroupVersion),
+	}
+
+	for _, apiServiceInfo := range s.apiServiceInfo {
+		if apiServiceInfo.discovery == nil {
+			continue
+		}
+
+		for key, item := range apiServiceInfo.discovery.Paths {
+			merged.Paths[key] = item
+		}
+	}
+	return merged
+}
+
+// handleDiscovery is the handler for OpenAPI V3 Discovery
+func (s *specProxier) handleDiscovery(w http.ResponseWriter, r *http.Request) {
+	merged := s.getOpenAPIV3Root()
+	j, err := json.Marshal(&merged)
+	if err != nil {
+		w.WriteHeader(500)
+		klog.Errorf("failed to created merged OpenAPIv3 discovery response: %s", err.Error())
+		return
+	}
+
+	http.ServeContent(w, r, "/openapi/v3", time.Now(), bytes.NewReader(j))
+}
+
+// handleGroupVersion is the OpenAPI V3 handler for a specified group/version
+func (s *specProxier) handleGroupVersion(w http.ResponseWriter, r *http.Request) {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	// TODO: Import this logic from kube-openapi instead of duplicating
+	// URLs for OpenAPI V3 have the format /openapi/v3/<groupversionpath>
+	// SplitAfterN with 4 yields ["", "openapi", "v3", <groupversionpath>]
+	url := strings.SplitAfterN(r.URL.Path, "/", 4)
+	targetGV := url[3]
+
+	for _, apiServiceInfo := range s.apiServiceInfo {
+		if apiServiceInfo.discovery == nil {
+			continue
+		}
+
+		for key := range apiServiceInfo.discovery.Paths {
+			if targetGV == key {
+				apiServiceInfo.handler.ServeHTTP(w, r)
+				return
+			}
+		}
+	}
+	// No group-versions match the desired request
+	w.WriteHeader(404)
+}
+
+// Register registers the OpenAPI V3 Discovery and GroupVersion handlers
+func (s *specProxier) register(handler common.PathHandlerByGroupVersion) {
+	handler.Handle("/openapi/v3", http.HandlerFunc(s.handleDiscovery))
+	handler.HandlePrefix("/openapi/v3/", http.HandlerFunc(s.handleGroupVersion))
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/downloader.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/kube-openapi/pkg/handler3"
+)
+
+type NotFoundError struct {
+}
+
+func (e *NotFoundError) Error() string {
+	return ""
+}
+
+// Downloader is the OpenAPI downloader type. It will try to download spec from /openapi/v3 and /openap/v3/<group>/<version> endpoints.
+type Downloader struct {
+}
+
+// NewDownloader creates a new OpenAPI Downloader.
+func NewDownloader() Downloader {
+	return Downloader{}
+}
+
+func (s *Downloader) handlerWithUser(handler http.Handler, info user.Info) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		req = req.WithContext(request.WithUser(req.Context(), info))
+		handler.ServeHTTP(w, req)
+	})
+}
+
+// OpenAPIV3Root downloads the OpenAPI V3 root document from an APIService
+func (s *Downloader) OpenAPIV3Root(handler http.Handler) (*handler3.OpenAPIV3Discovery, int, error) {
+	handler = s.handlerWithUser(handler, &user.DefaultInfo{Name: aggregatorUser})
+	handler = http.TimeoutHandler(handler, specDownloadTimeout, "request timed out")
+
+	req, err := http.NewRequest("GET", "/openapi/v3", nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	writer := newInMemoryResponseWriter()
+	handler.ServeHTTP(writer, req)
+
+	switch writer.respCode {
+	case http.StatusNotFound:
+		return nil, writer.respCode, nil
+	case http.StatusOK:
+		groups := handler3.OpenAPIV3Discovery{}
+		if err := json.Unmarshal(writer.data, &groups); err != nil {
+			return nil, writer.respCode, err
+		}
+		return &groups, writer.respCode, nil
+	}
+	return nil, writer.respCode, fmt.Errorf("Error, could not get list of group versions for APIService")
+}
+
+// inMemoryResponseWriter is a http.Writer that keep the response in memory.
+type inMemoryResponseWriter struct {
+	writeHeaderCalled bool
+	header            http.Header
+	respCode          int
+	data              []byte
+}
+
+func newInMemoryResponseWriter() *inMemoryResponseWriter {
+	return &inMemoryResponseWriter{header: http.Header{}}
+}
+
+func (r *inMemoryResponseWriter) Header() http.Header {
+	return r.header
+}
+
+func (r *inMemoryResponseWriter) WriteHeader(code int) {
+	r.writeHeaderCalled = true
+	r.respCode = code
+}
+
+func (r *inMemoryResponseWriter) Write(in []byte) (int, error) {
+	if !r.writeHeaderCalled {
+		r.WriteHeader(http.StatusOK)
+	}
+	r.data = append(r.data, in...)
+	return len(in), nil
+}
+
+func (r *inMemoryResponseWriter) String() string {
+	s := fmt.Sprintf("ResponseCode: %d", r.respCode)
+	if r.data != nil {
+		s += fmt.Sprintf(", Body: %s", string(r.data))
+	}
+	if r.header != nil {
+		s += fmt.Sprintf(", Header: %s", r.header)
+	}
+	return s
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/openapiv3/controller.go

@@ -0,0 +1,174 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapiv3
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator"
+)
+
+const (
+	successfulUpdateDelay      = time.Minute
+	successfulUpdateDelayLocal = time.Second
+	failedUpdateMaxExpDelay    = time.Hour
+)
+
+type syncAction int
+
+const (
+	syncRequeue syncAction = iota
+	syncRequeueRateLimited
+	syncNothing
+)
+
+// AggregationController periodically checks the list of group-versions handled by each APIService and updates the discovery page periodically
+type AggregationController struct {
+	openAPIAggregationManager aggregator.SpecProxier
+	queue                     workqueue.RateLimitingInterface
+
+	// To allow injection for testing.
+	syncHandler func(key string) (syncAction, error)
+}
+
+// NewAggregationController creates new OpenAPI aggregation controller.
+func NewAggregationController(openAPIAggregationManager aggregator.SpecProxier) *AggregationController {
+	c := &AggregationController{
+		openAPIAggregationManager: openAPIAggregationManager,
+		queue: workqueue.NewNamedRateLimitingQueue(
+			workqueue.NewItemExponentialFailureRateLimiter(successfulUpdateDelay, failedUpdateMaxExpDelay),
+			"open_api_v3_aggregation_controller",
+		),
+	}
+
+	c.syncHandler = c.sync
+
+	// update each service at least once, also those which are not coming from APIServices, namely local services
+	for _, name := range openAPIAggregationManager.GetAPIServiceNames() {
+		c.queue.AddAfter(name, time.Second)
+	}
+
+	return c
+}
+
+// Run starts OpenAPI AggregationController
+func (c *AggregationController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting OpenAPI V3 AggregationController")
+	defer klog.Info("Shutting down OpenAPI V3 AggregationController")
+
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+func (c *AggregationController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *AggregationController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	defer c.queue.Done(key)
+	if quit {
+		return false
+	}
+
+	if aggregator.IsLocalAPIService(key.(string)) {
+		// for local delegation targets that are aggregated once per second, log at
+		// higher level to avoid flooding the log
+		klog.V(6).Infof("OpenAPI AggregationController: Processing item %s", key)
+	} else {
+		klog.V(4).Infof("OpenAPI AggregationController: Processing item %s", key)
+	}
+
+	action, err := c.syncHandler(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+	} else {
+		utilruntime.HandleError(fmt.Errorf("loading OpenAPI spec for %q failed with: %v", key, err))
+	}
+
+	switch action {
+	case syncRequeue:
+		if aggregator.IsLocalAPIService(key.(string)) {
+			klog.V(7).Infof("OpenAPI AggregationController: action for local item %s: Requeue after %s.", key, successfulUpdateDelayLocal)
+			c.queue.AddAfter(key, successfulUpdateDelayLocal)
+		} else {
+			klog.V(7).Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
+			c.queue.AddAfter(key, successfulUpdateDelay)
+		}
+	case syncRequeueRateLimited:
+		klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key)
+		c.queue.AddRateLimited(key)
+	case syncNothing:
+		klog.Infof("OpenAPI AggregationController: action for item %s: Nothing (removed from the queue).", key)
+	}
+
+	return true
+}
+
+func (c *AggregationController) sync(key string) (syncAction, error) {
+	err := c.openAPIAggregationManager.UpdateAPIServiceSpec(key)
+	switch {
+	case err != nil:
+		return syncRequeueRateLimited, err
+	}
+	return syncRequeue, nil
+}
+
+// AddAPIService adds a new API Service to OpenAPI Aggregation.
+func (c *AggregationController) AddAPIService(handler http.Handler, apiService *v1.APIService) {
+	if apiService.Spec.Service == nil {
+		return
+	}
+	c.openAPIAggregationManager.AddUpdateAPIService(handler, apiService)
+	c.queue.AddAfter(apiService.Name, time.Second)
+}
+
+// UpdateAPIService updates API Service's info and handler.
+func (c *AggregationController) UpdateAPIService(handler http.Handler, apiService *v1.APIService) {
+	if apiService.Spec.Service == nil {
+		return
+	}
+	c.openAPIAggregationManager.AddUpdateAPIService(handler, apiService)
+	key := apiService.Name
+	if c.queue.NumRequeues(key) > 0 {
+		// The item has failed before. Remove it from failure queue and
+		// update it in a second
+		c.queue.Forget(key)
+		c.queue.AddAfter(key, time.Second)
+	}
+}
+
+// RemoveAPIService removes API Service from OpenAPI Aggregation Controller.
+func (c *AggregationController) RemoveAPIService(apiServiceName string) {
+	c.openAPIAggregationManager.RemoveAPIServiceSpec(apiServiceName)
+	// This will only remove it if it was failing before. If it was successful, processNextWorkItem will figure it out
+	// and will not add it again to the queue.
+	c.queue.Forget(apiServiceName)
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go

@@ -0,0 +1,692 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"reflect"
+	"sync"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	v1informers "k8s.io/client-go/informers/core/v1"
+	v1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/transport"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
+	informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
+	listers "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers"
+)
+
+// making sure we only register metrics once into legacy registry
+var registerIntoLegacyRegistryOnce sync.Once
+
+type certKeyFunc func() ([]byte, []byte)
+
+// ServiceResolver knows how to convert a service reference into an actual location.
+type ServiceResolver interface {
+	ResolveEndpoint(namespace, name string, port int32) (*url.URL, error)
+}
+
+// AvailableConditionController handles checking the availability of registered API services.
+type AvailableConditionController struct {
+	apiServiceClient apiregistrationclient.APIServicesGetter
+
+	apiServiceLister listers.APIServiceLister
+	apiServiceSynced cache.InformerSynced
+
+	// serviceLister is used to get the IP to create the transport for
+	serviceLister  v1listers.ServiceLister
+	servicesSynced cache.InformerSynced
+
+	endpointsLister v1listers.EndpointsLister
+	endpointsSynced cache.InformerSynced
+
+	// dialContext specifies the dial function for creating unencrypted TCP connections.
+	dialContext                func(ctx context.Context, network, address string) (net.Conn, error)
+	proxyCurrentCertKeyContent certKeyFunc
+	serviceResolver            ServiceResolver
+
+	// To allow injection for testing.
+	syncFn func(key string) error
+
+	queue workqueue.RateLimitingInterface
+	// map from service-namespace -> service-name -> apiservice names
+	cache map[string]map[string][]string
+	// this lock protects operations on the above cache
+	cacheLock sync.RWMutex
+
+	// TLS config with customized dialer cannot be cached by the client-go
+	// tlsTransportCache. Use a local cache here to reduce the chance of
+	// the controller spamming idle connections with short-lived transports.
+	// NOTE: the cache works because we assume that the transports constructed
+	// by the controller only vary on the dynamic cert/key.
+	tlsCache *tlsTransportCache
+
+	// metrics registered into legacy registry
+	metrics *availabilityMetrics
+}
+
+type tlsTransportCache struct {
+	mu         sync.Mutex
+	transports map[tlsCacheKey]http.RoundTripper
+}
+
+func (c *tlsTransportCache) get(config *rest.Config) (http.RoundTripper, error) {
+	// If the available controller doesn't customzie the dialer (and we know from
+	// the code that the controller doesn't customzie other functions i.e. Proxy
+	// and GetCert (ExecProvider)), the config is cacheable by the client-go TLS
+	// transport cache. Let's skip the local cache and depend on the client-go cache.
+	if config.Dial == nil {
+		return rest.TransportFor(config)
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	// See if we already have a custom transport for this config
+	key := tlsConfigKey(config)
+	if t, ok := c.transports[key]; ok {
+		return t, nil
+	}
+	restTransport, err := rest.TransportFor(config)
+	if err != nil {
+		return nil, err
+	}
+	c.transports[key] = restTransport
+	return restTransport, nil
+}
+
+type tlsCacheKey struct {
+	certData string
+	keyData  string `datapolicy:"secret-key"`
+}
+
+func tlsConfigKey(c *rest.Config) tlsCacheKey {
+	return tlsCacheKey{
+		certData: string(c.TLSClientConfig.CertData),
+		keyData:  string(c.TLSClientConfig.KeyData),
+	}
+}
+
+// NewAvailableConditionController returns a new AvailableConditionController.
+func NewAvailableConditionController(
+	apiServiceInformer informers.APIServiceInformer,
+	serviceInformer v1informers.ServiceInformer,
+	endpointsInformer v1informers.EndpointsInformer,
+	apiServiceClient apiregistrationclient.APIServicesGetter,
+	proxyTransport *http.Transport,
+	proxyCurrentCertKeyContent certKeyFunc,
+	serviceResolver ServiceResolver,
+	egressSelector *egressselector.EgressSelector,
+) (*AvailableConditionController, error) {
+	c := &AvailableConditionController{
+		apiServiceClient: apiServiceClient,
+		apiServiceLister: apiServiceInformer.Lister(),
+		apiServiceSynced: apiServiceInformer.Informer().HasSynced,
+		serviceLister:    serviceInformer.Lister(),
+		servicesSynced:   serviceInformer.Informer().HasSynced,
+		endpointsLister:  endpointsInformer.Lister(),
+		endpointsSynced:  endpointsInformer.Informer().HasSynced,
+		serviceResolver:  serviceResolver,
+		queue: workqueue.NewNamedRateLimitingQueue(
+			// We want a fairly tight requeue time.  The controller listens to the API, but because it relies on the routability of the
+			// service network, it is possible for an external, non-watchable factor to affect availability.  This keeps
+			// the maximum disruption time to a minimum, but it does prevent hot loops.
+			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 30*time.Second),
+			"AvailableConditionController"),
+		proxyCurrentCertKeyContent: proxyCurrentCertKeyContent,
+		tlsCache:                   &tlsTransportCache{transports: make(map[tlsCacheKey]http.RoundTripper)},
+		metrics:                    newAvailabilityMetrics(),
+	}
+
+	if egressSelector != nil {
+		networkContext := egressselector.Cluster.AsNetworkContext()
+		var egressDialer utilnet.DialFunc
+		egressDialer, err := egressSelector.Lookup(networkContext)
+		if err != nil {
+			return nil, err
+		}
+		c.dialContext = egressDialer
+	} else if proxyTransport != nil && proxyTransport.DialContext != nil {
+		c.dialContext = proxyTransport.DialContext
+	}
+
+	// resync on this one because it is low cardinality and rechecking the actual discovery
+	// allows us to detect health in a more timely fashion when network connectivity to
+	// nodes is snipped, but the network still attempts to route there.  See
+	// https://github.com/openshift/origin/issues/17159#issuecomment-341798063
+	apiServiceInformer.Informer().AddEventHandlerWithResyncPeriod(
+		cache.ResourceEventHandlerFuncs{
+			AddFunc:    c.addAPIService,
+			UpdateFunc: c.updateAPIService,
+			DeleteFunc: c.deleteAPIService,
+		},
+		30*time.Second)
+
+	serviceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addService,
+		UpdateFunc: c.updateService,
+		DeleteFunc: c.deleteService,
+	})
+
+	endpointsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addEndpoints,
+		UpdateFunc: c.updateEndpoints,
+		DeleteFunc: c.deleteEndpoints,
+	})
+
+	c.syncFn = c.sync
+
+	// TODO: decouple from legacyregistry
+	var err error
+	registerIntoLegacyRegistryOnce.Do(func() {
+		err = c.metrics.Register(legacyregistry.Register, legacyregistry.CustomRegister)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (c *AvailableConditionController) sync(key string) error {
+	originalAPIService, err := c.apiServiceLister.Get(key)
+	if apierrors.IsNotFound(err) {
+		c.metrics.ForgetAPIService(key)
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	// if a particular transport was specified, use that otherwise build one
+	// construct an http client that will ignore TLS verification (if someone owns the network and messes with your status
+	// that's not so bad) and sets a very short timeout.  This is a best effort GET that provides no additional information
+	restConfig := &rest.Config{
+		TLSClientConfig: rest.TLSClientConfig{
+			Insecure: true,
+		},
+	}
+
+	if c.proxyCurrentCertKeyContent != nil {
+		proxyClientCert, proxyClientKey := c.proxyCurrentCertKeyContent()
+
+		restConfig.TLSClientConfig.CertData = proxyClientCert
+		restConfig.TLSClientConfig.KeyData = proxyClientKey
+	}
+	if c.dialContext != nil {
+		restConfig.Dial = c.dialContext
+	}
+	// TLS config with customized dialer cannot be cached by the client-go
+	// tlsTransportCache. Use a local cache here to reduce the chance of
+	// the controller spamming idle connections with short-lived transports.
+	// NOTE: the cache works because we assume that the transports constructed
+	// by the controller only vary on the dynamic cert/key.
+	restTransport, err := c.tlsCache.get(restConfig)
+	if err != nil {
+		return err
+	}
+	discoveryClient := &http.Client{
+		Transport: restTransport,
+		// the request should happen quickly.
+		Timeout: 5 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	apiService := originalAPIService.DeepCopy()
+
+	availableCondition := apiregistrationv1.APIServiceCondition{
+		Type:               apiregistrationv1.Available,
+		Status:             apiregistrationv1.ConditionTrue,
+		LastTransitionTime: metav1.Now(),
+	}
+
+	// local API services are always considered available
+	if apiService.Spec.Service == nil {
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, apiregistrationv1apihelper.NewLocalAvailableAPIServiceCondition())
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	}
+
+	service, err := c.serviceLister.Services(apiService.Spec.Service.Namespace).Get(apiService.Spec.Service.Name)
+	if apierrors.IsNotFound(err) {
+		availableCondition.Status = apiregistrationv1.ConditionFalse
+		availableCondition.Reason = "ServiceNotFound"
+		availableCondition.Message = fmt.Sprintf("service/%s in %q is not present", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace)
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	} else if err != nil {
+		availableCondition.Status = apiregistrationv1.ConditionUnknown
+		availableCondition.Reason = "ServiceAccessError"
+		availableCondition.Message = fmt.Sprintf("service/%s in %q cannot be checked due to: %v", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace, err)
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	}
+
+	if service.Spec.Type == v1.ServiceTypeClusterIP {
+		// if we have a cluster IP service, it must be listening on configured port and we can check that
+		servicePort := apiService.Spec.Service.Port
+		portName := ""
+		foundPort := false
+		for _, port := range service.Spec.Ports {
+			if port.Port == *servicePort {
+				foundPort = true
+				portName = port.Name
+				break
+			}
+		}
+		if !foundPort {
+			availableCondition.Status = apiregistrationv1.ConditionFalse
+			availableCondition.Reason = "ServicePortError"
+			availableCondition.Message = fmt.Sprintf("service/%s in %q is not listening on port %d", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace, *apiService.Spec.Service.Port)
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+			return err
+		}
+
+		endpoints, err := c.endpointsLister.Endpoints(apiService.Spec.Service.Namespace).Get(apiService.Spec.Service.Name)
+		if apierrors.IsNotFound(err) {
+			availableCondition.Status = apiregistrationv1.ConditionFalse
+			availableCondition.Reason = "EndpointsNotFound"
+			availableCondition.Message = fmt.Sprintf("cannot find endpoints for service/%s in %q", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace)
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+			return err
+		} else if err != nil {
+			availableCondition.Status = apiregistrationv1.ConditionUnknown
+			availableCondition.Reason = "EndpointsAccessError"
+			availableCondition.Message = fmt.Sprintf("service/%s in %q cannot be checked due to: %v", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace, err)
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+			return err
+		}
+		hasActiveEndpoints := false
+	outer:
+		for _, subset := range endpoints.Subsets {
+			if len(subset.Addresses) == 0 {
+				continue
+			}
+			for _, endpointPort := range subset.Ports {
+				if endpointPort.Name == portName {
+					hasActiveEndpoints = true
+					break outer
+				}
+			}
+		}
+		if !hasActiveEndpoints {
+			availableCondition.Status = apiregistrationv1.ConditionFalse
+			availableCondition.Reason = "MissingEndpoints"
+			availableCondition.Message = fmt.Sprintf("endpoints for service/%s in %q have no addresses with port name %q", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace, portName)
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+			return err
+		}
+	}
+	// actually try to hit the discovery endpoint when it isn't local and when we're routing as a service.
+	if apiService.Spec.Service != nil && c.serviceResolver != nil {
+		attempts := 5
+		results := make(chan error, attempts)
+		for i := 0; i < attempts; i++ {
+			go func() {
+				discoveryURL, err := c.serviceResolver.ResolveEndpoint(apiService.Spec.Service.Namespace, apiService.Spec.Service.Name, *apiService.Spec.Service.Port)
+				if err != nil {
+					results <- err
+					return
+				}
+				// render legacyAPIService health check path when it is delegated to a service
+				if apiService.Name == "v1." {
+					discoveryURL.Path = "/api/" + apiService.Spec.Version
+				} else {
+					discoveryURL.Path = "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
+				}
+
+				errCh := make(chan error, 1)
+				go func() {
+					// be sure to check a URL that the aggregated API server is required to serve
+					newReq, err := http.NewRequest("GET", discoveryURL.String(), nil)
+					if err != nil {
+						errCh <- err
+						return
+					}
+
+					// setting the system-masters identity ensures that we will always have access rights
+					transport.SetAuthProxyHeaders(newReq, "system:kube-aggregator", []string{"system:masters"}, nil)
+					resp, err := discoveryClient.Do(newReq)
+					if resp != nil {
+						resp.Body.Close()
+						// we should always been in the 200s or 300s
+						if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+							errCh <- fmt.Errorf("bad status from %v: %v", discoveryURL, resp.StatusCode)
+							return
+						}
+					}
+
+					errCh <- err
+				}()
+
+				select {
+				case err = <-errCh:
+					if err != nil {
+						results <- fmt.Errorf("failing or missing response from %v: %v", discoveryURL, err)
+						return
+					}
+
+					// we had trouble with slow dial and DNS responses causing us to wait too long.
+					// we added this as insurance
+				case <-time.After(6 * time.Second):
+					results <- fmt.Errorf("timed out waiting for %v", discoveryURL)
+					return
+				}
+
+				results <- nil
+			}()
+		}
+
+		var lastError error
+		for i := 0; i < attempts; i++ {
+			lastError = <-results
+			// if we had at least one success, we are successful overall and we can return now
+			if lastError == nil {
+				break
+			}
+		}
+
+		if lastError != nil {
+			availableCondition.Status = apiregistrationv1.ConditionFalse
+			availableCondition.Reason = "FailedDiscoveryCheck"
+			availableCondition.Message = lastError.Error()
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, updateErr := c.updateAPIServiceStatus(originalAPIService, apiService)
+			if updateErr != nil {
+				return updateErr
+			}
+			// force a requeue to make it very obvious that this will be retried at some point in the future
+			// along with other requeues done via service change, endpoint change, and resync
+			return lastError
+		}
+	}
+
+	availableCondition.Reason = "Passed"
+	availableCondition.Message = "all checks passed"
+	apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+	_, err = c.updateAPIServiceStatus(originalAPIService, apiService)
+	return err
+}
+
+// updateAPIServiceStatus only issues an update if a change is detected.  We have a tight resync loop to quickly detect dead
+// apiservices. Doing that means we don't want to quickly issue no-op updates.
+func (c *AvailableConditionController) updateAPIServiceStatus(originalAPIService, newAPIService *apiregistrationv1.APIService) (*apiregistrationv1.APIService, error) {
+	// update this metric on every sync operation to reflect the actual state
+	c.setUnavailableGauge(newAPIService)
+
+	if equality.Semantic.DeepEqual(originalAPIService.Status, newAPIService.Status) {
+		return newAPIService, nil
+	}
+
+	orig := apiregistrationv1apihelper.GetAPIServiceConditionByType(originalAPIService, apiregistrationv1.Available)
+	now := apiregistrationv1apihelper.GetAPIServiceConditionByType(newAPIService, apiregistrationv1.Available)
+	unknown := apiregistrationv1.APIServiceCondition{
+		Type:   apiregistrationv1.Available,
+		Status: apiregistrationv1.ConditionUnknown,
+	}
+	if orig == nil {
+		orig = &unknown
+	}
+	if now == nil {
+		now = &unknown
+	}
+	if *orig != *now {
+		klog.V(2).InfoS("changing APIService availability", "name", newAPIService.Name, "oldStatus", orig.Status, "newStatus", now.Status, "message", now.Message, "reason", now.Reason)
+	}
+
+	newAPIService, err := c.apiServiceClient.APIServices().UpdateStatus(context.TODO(), newAPIService, metav1.UpdateOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	c.setUnavailableCounter(originalAPIService, newAPIService)
+	return newAPIService, nil
+}
+
+// Run starts the AvailableConditionController loop which manages the availability condition of API services.
+func (c *AvailableConditionController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting AvailableConditionController")
+	defer klog.Info("Shutting down AvailableConditionController")
+
+	if !controllers.WaitForCacheSync("AvailableConditionController", stopCh, c.apiServiceSynced, c.servicesSynced, c.endpointsSynced) {
+		return
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.runWorker, time.Second, stopCh)
+	}
+
+	<-stopCh
+}
+
+func (c *AvailableConditionController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *AvailableConditionController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	err := c.syncFn(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *AvailableConditionController) addAPIService(obj interface{}) {
+	castObj := obj.(*apiregistrationv1.APIService)
+	klog.V(4).Infof("Adding %s", castObj.Name)
+	if castObj.Spec.Service != nil {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(castObj.Name)
+}
+
+func (c *AvailableConditionController) updateAPIService(oldObj, newObj interface{}) {
+	castObj := newObj.(*apiregistrationv1.APIService)
+	oldCastObj := oldObj.(*apiregistrationv1.APIService)
+	klog.V(4).Infof("Updating %s", oldCastObj.Name)
+	if !reflect.DeepEqual(castObj.Spec.Service, oldCastObj.Spec.Service) {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(oldCastObj.Name)
+}
+
+func (c *AvailableConditionController) deleteAPIService(obj interface{}) {
+	castObj, ok := obj.(*apiregistrationv1.APIService)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*apiregistrationv1.APIService)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	klog.V(4).Infof("Deleting %q", castObj.Name)
+	if castObj.Spec.Service != nil {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(castObj.Name)
+}
+
+func (c *AvailableConditionController) getAPIServicesFor(obj runtime.Object) []string {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return nil
+	}
+	c.cacheLock.RLock()
+	defer c.cacheLock.RUnlock()
+	return c.cache[metadata.GetNamespace()][metadata.GetName()]
+}
+
+// if the service/endpoint handler wins the race against the cache rebuilding, it may queue a no-longer-relevant apiservice
+// (which will get processed an extra time - this doesn't matter),
+// and miss a newly relevant apiservice (which will get queued by the apiservice handler)
+func (c *AvailableConditionController) rebuildAPIServiceCache() {
+	apiServiceList, _ := c.apiServiceLister.List(labels.Everything())
+	newCache := map[string]map[string][]string{}
+	for _, apiService := range apiServiceList {
+		if apiService.Spec.Service == nil {
+			continue
+		}
+		if newCache[apiService.Spec.Service.Namespace] == nil {
+			newCache[apiService.Spec.Service.Namespace] = map[string][]string{}
+		}
+		newCache[apiService.Spec.Service.Namespace][apiService.Spec.Service.Name] = append(newCache[apiService.Spec.Service.Namespace][apiService.Spec.Service.Name], apiService.Name)
+	}
+
+	c.cacheLock.Lock()
+	defer c.cacheLock.Unlock()
+	c.cache = newCache
+}
+
+// TODO, think of a way to avoid checking on every service manipulation
+
+func (c *AvailableConditionController) addService(obj interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v1.Service)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) updateService(obj, _ interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v1.Service)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) deleteService(obj interface{}) {
+	castObj, ok := obj.(*v1.Service)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*v1.Service)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	for _, apiService := range c.getAPIServicesFor(castObj) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) addEndpoints(obj interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v1.Endpoints)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) updateEndpoints(obj, _ interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v1.Endpoints)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) deleteEndpoints(obj interface{}) {
+	castObj, ok := obj.(*v1.Endpoints)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*v1.Endpoints)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	for _, apiService := range c.getAPIServicesFor(castObj) {
+		c.queue.Add(apiService)
+	}
+}
+
+// setUnavailableGauge set the metrics so that it reflect the current state base on availability of the given service
+func (c *AvailableConditionController) setUnavailableGauge(newAPIService *apiregistrationv1.APIService) {
+	if apiregistrationv1apihelper.IsAPIServiceConditionTrue(newAPIService, apiregistrationv1.Available) {
+		c.metrics.SetAPIServiceAvailable(newAPIService.Name)
+		return
+	}
+
+	c.metrics.SetAPIServiceUnavailable(newAPIService.Name)
+}
+
+// setUnavailableCounter increases the metrics only if the given service is unavailable and its APIServiceCondition has changed
+func (c *AvailableConditionController) setUnavailableCounter(originalAPIService, newAPIService *apiregistrationv1.APIService) {
+	wasAvailable := apiregistrationv1apihelper.IsAPIServiceConditionTrue(originalAPIService, apiregistrationv1.Available)
+	isAvailable := apiregistrationv1apihelper.IsAPIServiceConditionTrue(newAPIService, apiregistrationv1.Available)
+	statusChanged := isAvailable != wasAvailable
+
+	if statusChanged && !isAvailable {
+		reason := "UnknownReason"
+		if newCondition := apiregistrationv1apihelper.GetAPIServiceConditionByType(newAPIService, apiregistrationv1.Available); newCondition != nil {
+			reason = newCondition.Reason
+		}
+		c.metrics.UnavailableCounter(newAPIService.Name, reason).Inc()
+	}
+}

vendor/k8s.io/kube-aggregator/pkg/controllers/status/metrics.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"sync"
+
+	"k8s.io/component-base/metrics"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	unavailableGaugeDesc = metrics.NewDesc(
+		"aggregator_unavailable_apiservice",
+		"Gauge of APIServices which are marked as unavailable broken down by APIService name.",
+		[]string{"name"},
+		nil,
+		metrics.ALPHA,
+		"",
+	)
+)
+
+type availabilityMetrics struct {
+	unavailableCounter *metrics.CounterVec
+
+	*availabilityCollector
+}
+
+func newAvailabilityMetrics() *availabilityMetrics {
+	return &availabilityMetrics{
+		unavailableCounter: metrics.NewCounterVec(
+			&metrics.CounterOpts{
+				Name:           "aggregator_unavailable_apiservice_total",
+				Help:           "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.",
+				StabilityLevel: metrics.ALPHA,
+			},
+			[]string{"name", "reason"},
+		),
+		availabilityCollector: newAvailabilityCollector(),
+	}
+}
+
+// Register registers apiservice availability metrics.
+func (m *availabilityMetrics) Register(
+	registrationFunc func(metrics.Registerable) error,
+	customRegistrationFunc func(metrics.StableCollector) error,
+) error {
+	err := registrationFunc(m.unavailableCounter)
+	if err != nil {
+		return err
+	}
+
+	err = customRegistrationFunc(m.availabilityCollector)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UnavailableCounter returns a counter to track apiservices marked as unavailable.
+func (m *availabilityMetrics) UnavailableCounter(apiServiceName, reason string) metrics.CounterMetric {
+	return m.unavailableCounter.WithLabelValues(apiServiceName, reason)
+}
+
+type availabilityCollector struct {
+	metrics.BaseStableCollector
+
+	mtx            sync.RWMutex
+	availabilities map[string]bool
+}
+
+// Check if apiServiceStatusCollector implements necessary interface.
+var _ metrics.StableCollector = &availabilityCollector{}
+
+func newAvailabilityCollector() *availabilityCollector {
+	return &availabilityCollector{
+		availabilities: make(map[string]bool),
+	}
+}
+
+// DescribeWithStability implements the metrics.StableCollector interface.
+func (c *availabilityCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
+	ch <- unavailableGaugeDesc
+}
+
+// CollectWithStability implements the metrics.StableCollector interface.
+func (c *availabilityCollector) CollectWithStability(ch chan<- metrics.Metric) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+
+	for apiServiceName, isAvailable := range c.availabilities {
+		gaugeValue := 1.0
+		if isAvailable {
+			gaugeValue = 0.0
+		}
+		ch <- metrics.NewLazyConstMetric(
+			unavailableGaugeDesc,
+			metrics.GaugeValue,
+			gaugeValue,
+			apiServiceName,
+		)
+	}
+}
+
+// SetAPIServiceAvailable sets the given apiservice availability gauge to available.
+func (c *availabilityCollector) SetAPIServiceAvailable(apiServiceKey string) {
+	c.setAPIServiceAvailability(apiServiceKey, true)
+}
+
+// SetAPIServiceUnavailable sets the given apiservice availability gauge to unavailable.
+func (c *availabilityCollector) SetAPIServiceUnavailable(apiServiceKey string) {
+	c.setAPIServiceAvailability(apiServiceKey, false)
+}
+
+func (c *availabilityCollector) setAPIServiceAvailability(apiServiceKey string, availability bool) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	c.availabilities[apiServiceKey] = availability
+}
+
+// ForgetAPIService removes the availability gauge of the given apiservice.
+func (c *availabilityCollector) ForgetAPIService(apiServiceKey string) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	delete(c.availabilities, apiServiceKey)
+}

vendor/k8s.io/kube-aggregator/pkg/registry/apiservice/etcd/etcd.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metatable "k8s.io/apimachinery/pkg/api/meta/table"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+	"k8s.io/kube-aggregator/pkg/registry/apiservice"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+)
+
+// REST implements a RESTStorage for API services against etcd
+type REST struct {
+	*genericregistry.Store
+}
+
+// NewREST returns a RESTStorage object that will work against API services.
+func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) *REST {
+	strategy := apiservice.NewStrategy(scheme)
+	store := &genericregistry.Store{
+		NewFunc:                  func() runtime.Object { return &apiregistration.APIService{} },
+		NewListFunc:              func() runtime.Object { return &apiregistration.APIServiceList{} },
+		PredicateFunc:            apiservice.MatchAPIService,
+		DefaultQualifiedResource: apiregistration.Resource("apiservices"),
+
+		CreateStrategy:      strategy,
+		UpdateStrategy:      strategy,
+		DeleteStrategy:      strategy,
+		ResetFieldsStrategy: strategy,
+
+		// TODO: define table converter that exposes more than name/creation timestamp
+		TableConvertor: rest.NewDefaultTableConvertor(apiregistration.Resource("apiservices")),
+	}
+	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: apiservice.GetAttrs}
+	if err := store.CompleteWithOptions(options); err != nil {
+		panic(err) // TODO: Propagate error up
+	}
+	return &REST{store}
+}
+
+// Implement CategoriesProvider
+var _ rest.CategoriesProvider = &REST{}
+
+// Categories implements the CategoriesProvider interface. Returns a list of categories a resource is part of.
+func (c *REST) Categories() []string {
+	return []string{"api-extensions"}
+}
+
+var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc()
+
+// ConvertToTable implements the TableConvertor interface for REST.
+func (c *REST) ConvertToTable(ctx context.Context, obj runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
+	table := &metav1.Table{
+		ColumnDefinitions: []metav1.TableColumnDefinition{
+			{Name: "Name", Type: "string", Format: "name", Description: swaggerMetadataDescriptions["name"]},
+			{Name: "Service", Type: "string", Description: "The reference to the service that hosts this API endpoint."},
+			{Name: "Available", Type: "string", Description: "Whether this service is available."},
+			{Name: "Age", Type: "string", Description: swaggerMetadataDescriptions["creationTimestamp"]},
+		},
+	}
+	if m, err := meta.ListAccessor(obj); err == nil {
+		table.ResourceVersion = m.GetResourceVersion()
+		table.Continue = m.GetContinue()
+		table.RemainingItemCount = m.GetRemainingItemCount()
+	} else {
+		if m, err := meta.CommonAccessor(obj); err == nil {
+			table.ResourceVersion = m.GetResourceVersion()
+		}
+	}
+
+	var err error
+	table.Rows, err = metatable.MetaToTableRow(obj, func(obj runtime.Object, m metav1.Object, name, age string) ([]interface{}, error) {
+		svc := obj.(*apiregistration.APIService)
+		service := "Local"
+		if svc.Spec.Service != nil {
+			service = fmt.Sprintf("%s/%s", svc.Spec.Service.Namespace, svc.Spec.Service.Name)
+		}
+		status := string(apiregistration.ConditionUnknown)
+		if condition := getCondition(svc.Status.Conditions, "Available"); condition != nil {
+			switch {
+			case condition.Status == apiregistration.ConditionTrue:
+				status = string(condition.Status)
+			case len(condition.Reason) > 0:
+				status = fmt.Sprintf("%s (%s)", condition.Status, condition.Reason)
+			default:
+				status = string(condition.Status)
+			}
+		}
+		return []interface{}{name, service, status, age}, nil
+	})
+	return table, err
+}
+
+func getCondition(conditions []apiregistration.APIServiceCondition, conditionType apiregistration.APIServiceConditionType) *apiregistration.APIServiceCondition {
+	for i, condition := range conditions {
+		if condition.Type == conditionType {
+			return &conditions[i]
+		}
+	}
+	return nil
+}
+
+// NewStatusREST makes a RESTStorage for status that has more limited options.
+// It is based on the original REST so that we can share the same underlying store
+func NewStatusREST(scheme *runtime.Scheme, rest *REST) *StatusREST {
+	strategy := apiservice.NewStatusStrategy(scheme)
+	statusStore := *rest.Store
+	statusStore.CreateStrategy = nil
+	statusStore.DeleteStrategy = nil
+	statusStore.UpdateStrategy = strategy
+	statusStore.ResetFieldsStrategy = strategy
+	return &StatusREST{store: &statusStore}
+}
+
+// StatusREST implements the REST endpoint for changing the status of an APIService.
+type StatusREST struct {
+	store *genericregistry.Store
+}
+
+var _ = rest.Patcher(&StatusREST{})
+
+// New creates a new APIService object.
+func (r *StatusREST) New() runtime.Object {
+	return &apiregistration.APIService{}
+}
+
+// Destroy cleans up resources on shutdown.
+func (r *StatusREST) Destroy() {
+	// Given that underlying store is shared with REST,
+	// we don't destroy it here explicitly.
+}
+
+// Get retrieves the object from the storage. It is required to support Patch.
+func (r *StatusREST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
+	return r.store.Get(ctx, name, options)
+}
+
+// Update alters the status subset of an object.
+func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	// We are explicitly setting forceAllowCreate to false in the call to the underlying storage because
+	// subresources should never allow create on update.
+	return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options)
+}
+
+// GetResetFields implements rest.ResetFieldsStrategy
+func (r *StatusREST) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
+	return r.store.GetResetFields()
+}

vendor/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rest
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/registry/generic"
+	"k8s.io/apiserver/pkg/registry/rest"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
+
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	apiservicestorage "k8s.io/kube-aggregator/pkg/registry/apiservice/etcd"
+)
+
+// NewRESTStorage returns an APIGroupInfo object that will work against apiservice.
+func NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter, shouldServeBeta bool) genericapiserver.APIGroupInfo {
+	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName, aggregatorscheme.Scheme, metav1.ParameterCodec, aggregatorscheme.Codecs)
+
+	storage := map[string]rest.Storage{}
+
+	if resource := "apiservices"; apiResourceConfigSource.ResourceEnabled(v1.SchemeGroupVersion.WithResource(resource)) {
+		apiServiceREST := apiservicestorage.NewREST(aggregatorscheme.Scheme, restOptionsGetter)
+		storage[resource] = apiServiceREST
+		storage[resource+"/status"] = apiservicestorage.NewStatusREST(aggregatorscheme.Scheme, apiServiceREST)
+	}
+
+	if len(storage) > 0 {
+		apiGroupInfo.VersionedResourcesStorageMap["v1"] = storage
+	}
+
+	return apiGroupInfo
+}

vendor/k8s.io/kube-aggregator/pkg/registry/apiservice/strategy.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiservice
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/registry/generic"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/names"
+
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/validation"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+)
+
+type apiServerStrategy struct {
+	runtime.ObjectTyper
+	names.NameGenerator
+}
+
+// apiServerStrategy must implement rest.RESTCreateUpdateStrategy
+var _ rest.RESTCreateUpdateStrategy = apiServerStrategy{}
+var Strategy = apiServerStrategy{}
+
+// NewStrategy creates a new apiServerStrategy.
+func NewStrategy(typer runtime.ObjectTyper) rest.CreateUpdateResetFieldsStrategy {
+	return apiServerStrategy{typer, names.SimpleNameGenerator}
+}
+
+func (apiServerStrategy) NamespaceScoped() bool {
+	return false
+}
+
+func (apiServerStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
+	fields := map[fieldpath.APIVersion]*fieldpath.Set{
+		"apiregistration.k8s.io/v1": fieldpath.NewSet(
+			fieldpath.MakePathOrDie("status"),
+		),
+		"apiregistration.k8s.io/v1beta1": fieldpath.NewSet(
+			fieldpath.MakePathOrDie("status"),
+		),
+	}
+
+	return fields
+}
+
+func (apiServerStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
+	apiservice := obj.(*apiregistration.APIService)
+	apiservice.Status = apiregistration.APIServiceStatus{}
+
+	// mark local API services as immediately available on create
+	if apiservice.Spec.Service == nil {
+		apiregistration.SetAPIServiceCondition(apiservice, apiregistration.NewLocalAvailableAPIServiceCondition())
+	}
+}
+
+func (apiServerStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
+	newAPIService := obj.(*apiregistration.APIService)
+	oldAPIService := old.(*apiregistration.APIService)
+	newAPIService.Status = oldAPIService.Status
+}
+
+func (apiServerStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
+	return validation.ValidateAPIService(obj.(*apiregistration.APIService))
+}
+
+// WarningsOnCreate returns warnings for the creation of the given object.
+func (apiServerStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string {
+	return nil
+}
+
+func (apiServerStrategy) AllowCreateOnUpdate() bool {
+	return false
+}
+
+func (apiServerStrategy) AllowUnconditionalUpdate() bool {
+	return false
+}
+
+func (apiServerStrategy) Canonicalize(obj runtime.Object) {
+}
+
+func (apiServerStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
+	return validation.ValidateAPIServiceUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
+}
+
+// WarningsOnUpdate returns warnings for the given update.
+func (apiServerStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string {
+	return nil
+}
+
+type apiServerStatusStrategy struct {
+	runtime.ObjectTyper
+	names.NameGenerator
+}
+
+// NewStatusStrategy creates a new apiServerStatusStrategy.
+func NewStatusStrategy(typer runtime.ObjectTyper) rest.UpdateResetFieldsStrategy {
+	return apiServerStatusStrategy{typer, names.SimpleNameGenerator}
+}
+
+func (apiServerStatusStrategy) NamespaceScoped() bool {
+	return false
+}
+
+func (apiServerStatusStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
+	fields := map[fieldpath.APIVersion]*fieldpath.Set{
+		"apiregistration.k8s.io/v1": fieldpath.NewSet(
+			fieldpath.MakePathOrDie("spec"),
+			fieldpath.MakePathOrDie("metadata"),
+		),
+		"apiregistration.k8s.io/v1beta1": fieldpath.NewSet(
+			fieldpath.MakePathOrDie("spec"),
+			fieldpath.MakePathOrDie("metadata"),
+		),
+	}
+
+	return fields
+}
+
+func (apiServerStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
+	newAPIService := obj.(*apiregistration.APIService)
+	oldAPIService := old.(*apiregistration.APIService)
+	newAPIService.Spec = oldAPIService.Spec
+	newAPIService.Labels = oldAPIService.Labels
+	newAPIService.Annotations = oldAPIService.Annotations
+	newAPIService.Finalizers = oldAPIService.Finalizers
+	newAPIService.OwnerReferences = oldAPIService.OwnerReferences
+}
+
+func (apiServerStatusStrategy) AllowCreateOnUpdate() bool {
+	return false
+}
+
+func (apiServerStatusStrategy) AllowUnconditionalUpdate() bool {
+	return false
+}
+
+// Canonicalize normalizes the object after validation.
+func (apiServerStatusStrategy) Canonicalize(obj runtime.Object) {
+}
+
+// ValidateUpdate validates an update of apiServerStatusStrategy.
+func (apiServerStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
+	return validation.ValidateAPIServiceStatusUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
+}
+
+// WarningsOnUpdate returns warnings for the given update.
+func (apiServerStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string {
+	return nil
+}
+
+// GetAttrs returns the labels and fields of an API server for filtering purposes.
+func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
+	apiserver, ok := obj.(*apiregistration.APIService)
+	if !ok {
+		return nil, nil, fmt.Errorf("given object is not a APIService")
+	}
+	return labels.Set(apiserver.ObjectMeta.Labels), ToSelectableFields(apiserver), nil
+}
+
+// MatchAPIService is the filter used by the generic etcd backend to watch events
+// from etcd to clients of the apiserver only interested in specific labels/fields.
+func MatchAPIService(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
+	return storage.SelectionPredicate{
+		Label:    label,
+		Field:    field,
+		GetAttrs: GetAttrs,
+	}
+}
+
+// ToSelectableFields returns a field set that represents the object.
+func ToSelectableFields(obj *apiregistration.APIService) fields.Set {
+	return generic.ObjectMetaFieldsSet(&obj.ObjectMeta, true)
+}

vendor/k8s.io/kube-openapi/pkg/aggregator/aggregator.go

@@ -0,0 +1,377 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+
+	"k8s.io/kube-openapi/pkg/validation/spec"
+
+	"k8s.io/kube-openapi/pkg/schemamutation"
+	"k8s.io/kube-openapi/pkg/util"
+)
+
+const gvkKey = "x-kubernetes-group-version-kind"
+
+// usedDefinitionForSpec returns a map with all used definitions in the provided spec as keys and true as values.
+func usedDefinitionForSpec(root *spec.Swagger) map[string]bool {
+	usedDefinitions := map[string]bool{}
+	walkOnAllReferences(func(ref *spec.Ref) {
+		if refStr := ref.String(); refStr != "" && strings.HasPrefix(refStr, definitionPrefix) {
+			usedDefinitions[refStr[len(definitionPrefix):]] = true
+		}
+	}, root)
+	return usedDefinitions
+}
+
+// FilterSpecByPaths removes unnecessary paths and definitions used by those paths.
+// i.e. if a Path removed by this function, all definitions used by it and not used
+// anywhere else will also be removed.
+func FilterSpecByPaths(sp *spec.Swagger, keepPathPrefixes []string) {
+	*sp = *FilterSpecByPathsWithoutSideEffects(sp, keepPathPrefixes)
+}
+
+// FilterSpecByPathsWithoutSideEffects removes unnecessary paths and definitions used by those paths.
+// i.e. if a Path removed by this function, all definitions used by it and not used
+// anywhere else will also be removed.
+// It does not modify the input, but the output shares data structures with the input.
+func FilterSpecByPathsWithoutSideEffects(sp *spec.Swagger, keepPathPrefixes []string) *spec.Swagger {
+	if sp.Paths == nil {
+		return sp
+	}
+
+	// Walk all references to find all used definitions. This function
+	// want to only deal with unused definitions resulted from filtering paths.
+	// Thus a definition will be removed only if it has been used before but
+	// it is unused because of a path prune.
+	initialUsedDefinitions := usedDefinitionForSpec(sp)
+
+	// First remove unwanted paths
+	prefixes := util.NewTrie(keepPathPrefixes)
+	ret := *sp
+	ret.Paths = &spec.Paths{
+		VendorExtensible: sp.Paths.VendorExtensible,
+		Paths:            map[string]spec.PathItem{},
+	}
+	for path, pathItem := range sp.Paths.Paths {
+		if !prefixes.HasPrefix(path) {
+			continue
+		}
+		ret.Paths.Paths[path] = pathItem
+	}
+
+	// Walk all references to find all definition references.
+	usedDefinitions := usedDefinitionForSpec(&ret)
+
+	// Remove unused definitions
+	ret.Definitions = spec.Definitions{}
+	for k, v := range sp.Definitions {
+		if usedDefinitions[k] || !initialUsedDefinitions[k] {
+			ret.Definitions[k] = v
+		}
+	}
+
+	return &ret
+}
+
+type rename struct {
+	from, to string
+}
+
+// renameDefinition renames references, without mutating the input.
+// The output might share data structures with the input.
+func renameDefinition(s *spec.Swagger, renames map[string]string) *spec.Swagger {
+	refRenames := make(map[string]string, len(renames))
+	foundOne := false
+	for k, v := range renames {
+		refRenames[definitionPrefix+k] = definitionPrefix + v
+		if _, ok := s.Definitions[k]; ok {
+			foundOne = true
+		}
+	}
+
+	if !foundOne {
+		return s
+	}
+
+	ret := &spec.Swagger{}
+	*ret = *s
+
+	ret = schemamutation.ReplaceReferences(func(ref *spec.Ref) *spec.Ref {
+		refName := ref.String()
+		if newRef, found := refRenames[refName]; found {
+			ret := spec.MustCreateRef(newRef)
+			return &ret
+		}
+		return ref
+	}, ret)
+
+	renamedDefinitions := make(spec.Definitions, len(ret.Definitions))
+	for k, v := range ret.Definitions {
+		if newRef, found := renames[k]; found {
+			k = newRef
+		}
+		renamedDefinitions[k] = v
+	}
+	ret.Definitions = renamedDefinitions
+
+	return ret
+}
+
+// MergeSpecsIgnorePathConflict is the same as MergeSpecs except it will ignore any path
+// conflicts by keeping the paths of destination. It will rename definition conflicts.
+// The source is not mutated.
+func MergeSpecsIgnorePathConflict(dest, source *spec.Swagger) error {
+	return mergeSpecs(dest, source, true, true)
+}
+
+// MergeSpecsFailOnDefinitionConflict is differ from MergeSpecs as it fails if there is
+// a definition conflict.
+// The source is not mutated.
+func MergeSpecsFailOnDefinitionConflict(dest, source *spec.Swagger) error {
+	return mergeSpecs(dest, source, false, false)
+}
+
+// MergeSpecs copies paths and definitions from source to dest, rename definitions if needed.
+// dest will be mutated, and source will not be changed. It will fail on path conflicts.
+// The source is not mutated.
+func MergeSpecs(dest, source *spec.Swagger) error {
+	return mergeSpecs(dest, source, true, false)
+}
+
+// mergeSpecs merges source into dest while resolving conflicts.
+// The source is not mutated.
+func mergeSpecs(dest, source *spec.Swagger, renameModelConflicts, ignorePathConflicts bool) (err error) {
+	// Paths may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+	if source.Paths == nil {
+		// When a source spec does not have any path, that means none of the definitions
+		// are used thus we should not do anything
+		return nil
+	}
+	if dest.Paths == nil {
+		dest.Paths = &spec.Paths{}
+	}
+	if ignorePathConflicts {
+		keepPaths := []string{}
+		hasConflictingPath := false
+		for k := range source.Paths.Paths {
+			if _, found := dest.Paths.Paths[k]; !found {
+				keepPaths = append(keepPaths, k)
+			} else {
+				hasConflictingPath = true
+			}
+		}
+		if len(keepPaths) == 0 {
+			// There is nothing to merge. All paths are conflicting.
+			return nil
+		}
+		if hasConflictingPath {
+			source = FilterSpecByPathsWithoutSideEffects(source, keepPaths)
+		}
+	}
+
+	// Check for model conflicts and rename to make definitions conflict-free (modulo different GVKs)
+	usedNames := map[string]bool{}
+	for k := range dest.Definitions {
+		usedNames[k] = true
+	}
+	renames := map[string]string{}
+DEFINITIONLOOP:
+	for k, v := range source.Definitions {
+		existing, found := dest.Definitions[k]
+		if !found || deepEqualDefinitionsModuloGVKs(&existing, &v) {
+			// skip for now, we copy them after the rename loop
+			continue
+		}
+
+		if !renameModelConflicts {
+			return fmt.Errorf("model name conflict in merging OpenAPI spec: %s", k)
+		}
+
+		// Reuse previously renamed model if one exists
+		var newName string
+		i := 1
+		for found {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			existing, found = dest.Definitions[newName]
+			if found && deepEqualDefinitionsModuloGVKs(&existing, &v) {
+				renames[k] = newName
+				continue DEFINITIONLOOP
+			}
+		}
+
+		_, foundInSource := source.Definitions[newName]
+		for usedNames[newName] || foundInSource {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			_, foundInSource = source.Definitions[newName]
+		}
+		renames[k] = newName
+		usedNames[newName] = true
+	}
+	source = renameDefinition(source, renames)
+
+	// now without conflict (modulo different GVKs), copy definitions to dest
+	for k, v := range source.Definitions {
+		if existing, found := dest.Definitions[k]; !found {
+			if dest.Definitions == nil {
+				dest.Definitions = spec.Definitions{}
+			}
+			dest.Definitions[k] = v
+		} else if merged, changed, err := mergedGVKs(&existing, &v); err != nil {
+			return err
+		} else if changed {
+			existing.Extensions[gvkKey] = merged
+		}
+	}
+
+	// Check for path conflicts
+	for k, v := range source.Paths.Paths {
+		if _, found := dest.Paths.Paths[k]; found {
+			return fmt.Errorf("unable to merge: duplicated path %s", k)
+		}
+		// PathItem may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+		if dest.Paths.Paths == nil {
+			dest.Paths.Paths = map[string]spec.PathItem{}
+		}
+		dest.Paths.Paths[k] = v
+	}
+
+	return nil
+}
+
+// deepEqualDefinitionsModuloGVKs compares s1 and s2, but ignores the x-kubernetes-group-version-kind extension.
+func deepEqualDefinitionsModuloGVKs(s1, s2 *spec.Schema) bool {
+	if s1 == nil {
+		return s2 == nil
+	} else if s2 == nil {
+		return false
+	}
+	if !reflect.DeepEqual(s1.Extensions, s2.Extensions) {
+		for k, v := range s1.Extensions {
+			if k == gvkKey {
+				continue
+			}
+			if !reflect.DeepEqual(v, s2.Extensions[k]) {
+				return false
+			}
+		}
+		len1 := len(s1.Extensions)
+		len2 := len(s2.Extensions)
+		if _, found := s1.Extensions[gvkKey]; found {
+			len1--
+		}
+		if _, found := s2.Extensions[gvkKey]; found {
+			len2--
+		}
+		if len1 != len2 {
+			return false
+		}
+
+		if s1.Extensions != nil {
+			shallowCopy := *s1
+			s1 = &shallowCopy
+			s1.Extensions = nil
+		}
+		if s2.Extensions != nil {
+			shallowCopy := *s2
+			s2 = &shallowCopy
+			s2.Extensions = nil
+		}
+	}
+
+	return reflect.DeepEqual(s1, s2)
+}
+
+// mergedGVKs merges the x-kubernetes-group-version-kind slices and returns the result, and whether
+// s1's x-kubernetes-group-version-kind slice was changed at all.
+func mergedGVKs(s1, s2 *spec.Schema) (interface{}, bool, error) {
+	gvk1, found1 := s1.Extensions[gvkKey]
+	gvk2, found2 := s2.Extensions[gvkKey]
+
+	if !found1 {
+		return gvk2, found2, nil
+	}
+	if !found2 {
+		return gvk1, false, nil
+	}
+
+	slice1, ok := gvk1.([]interface{})
+	if !ok {
+		return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice1)
+	}
+	slice2, ok := gvk2.([]interface{})
+	if !ok {
+		return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice2)
+	}
+
+	ret := make([]interface{}, len(slice1), len(slice1)+len(slice2))
+	keys := make([]string, 0, len(slice1)+len(slice2))
+	copy(ret, slice1)
+	seen := make(map[string]bool, len(slice1))
+	for _, x := range slice1 {
+		gvk, ok := x.(map[string]interface{})
+		if !ok {
+			return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
+		}
+		k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
+		keys = append(keys, k)
+		seen[k] = true
+	}
+	changed := false
+	for _, x := range slice2 {
+		gvk, ok := x.(map[string]interface{})
+		if !ok {
+			return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
+		}
+		k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
+		if seen[k] {
+			continue
+		}
+		ret = append(ret, x)
+		keys = append(keys, k)
+		changed = true
+	}
+
+	if changed {
+		sort.Sort(byKeys{ret, keys})
+	}
+
+	return ret, changed, nil
+}
+
+type byKeys struct {
+	values []interface{}
+	keys   []string
+}
+
+func (b byKeys) Len() int {
+	return len(b.values)
+}
+
+func (b byKeys) Less(i, j int) bool {
+	return b.keys[i] < b.keys[j]
+}
+
+func (b byKeys) Swap(i, j int) {
+	b.values[i], b.values[j] = b.values[j], b.values[i]
+	b.keys[i], b.keys[j] = b.keys[j], b.keys[i]
+}

vendor/k8s.io/kube-openapi/pkg/aggregator/walker.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aggregator
+
+import (
+	"strings"
+
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+const (
+	definitionPrefix = "#/definitions/"
+)
+
+// Run a readonlyReferenceWalker method on all references of an OpenAPI spec
+type readonlyReferenceWalker struct {
+	// walkRefCallback will be called on each reference. The input will never be nil.
+	walkRefCallback func(ref *spec.Ref)
+
+	// The spec to walk through.
+	root *spec.Swagger
+}
+
+// walkOnAllReferences recursively walks on all references, while following references into definitions.
+// it calls walkRef on each found reference.
+func walkOnAllReferences(walkRef func(ref *spec.Ref), root *spec.Swagger) {
+	alreadyVisited := map[string]bool{}
+
+	walker := &readonlyReferenceWalker{
+		root: root,
+	}
+	walker.walkRefCallback = func(ref *spec.Ref) {
+		walkRef(ref)
+
+		refStr := ref.String()
+		if refStr == "" || !strings.HasPrefix(refStr, definitionPrefix) {
+			return
+		}
+		defName := refStr[len(definitionPrefix):]
+
+		if _, found := root.Definitions[defName]; found && !alreadyVisited[refStr] {
+			alreadyVisited[refStr] = true
+			def := root.Definitions[defName]
+			walker.walkSchema(&def)
+		}
+	}
+	walker.Start()
+}
+
+func (s *readonlyReferenceWalker) walkSchema(schema *spec.Schema) {
+	if schema == nil {
+		return
+	}
+	s.walkRefCallback(&schema.Ref)
+	var v *spec.Schema
+	if len(schema.Definitions)+len(schema.Properties)+len(schema.PatternProperties) > 0 {
+		v = &spec.Schema{}
+	}
+	for k := range schema.Definitions {
+		*v = schema.Definitions[k]
+		s.walkSchema(v)
+	}
+	for k := range schema.Properties {
+		*v = schema.Properties[k]
+		s.walkSchema(v)
+	}
+	for k := range schema.PatternProperties {
+		*v = schema.PatternProperties[k]
+		s.walkSchema(v)
+	}
+	for i := range schema.AllOf {
+		s.walkSchema(&schema.AllOf[i])
+	}
+	for i := range schema.AnyOf {
+		s.walkSchema(&schema.AnyOf[i])
+	}
+	for i := range schema.OneOf {
+		s.walkSchema(&schema.OneOf[i])
+	}
+	if schema.Not != nil {
+		s.walkSchema(schema.Not)
+	}
+	if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
+		s.walkSchema(schema.AdditionalProperties.Schema)
+	}
+	if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
+		s.walkSchema(schema.AdditionalItems.Schema)
+	}
+	if schema.Items != nil {
+		if schema.Items.Schema != nil {
+			s.walkSchema(schema.Items.Schema)
+		}
+		for i := range schema.Items.Schemas {
+			s.walkSchema(&schema.Items.Schemas[i])
+		}
+	}
+}
+
+func (s *readonlyReferenceWalker) walkParams(params []spec.Parameter) {
+	if params == nil {
+		return
+	}
+	for _, param := range params {
+		s.walkRefCallback(&param.Ref)
+		s.walkSchema(param.Schema)
+		if param.Items != nil {
+			s.walkRefCallback(&param.Items.Ref)
+		}
+	}
+}
+
+func (s *readonlyReferenceWalker) walkResponse(resp *spec.Response) {
+	if resp == nil {
+		return
+	}
+	s.walkRefCallback(&resp.Ref)
+	s.walkSchema(resp.Schema)
+}
+
+func (s *readonlyReferenceWalker) walkOperation(op *spec.Operation) {
+	if op == nil {
+		return
+	}
+	s.walkParams(op.Parameters)
+	if op.Responses == nil {
+		return
+	}
+	s.walkResponse(op.Responses.Default)
+	for _, r := range op.Responses.StatusCodeResponses {
+		s.walkResponse(&r)
+	}
+}
+
+func (s *readonlyReferenceWalker) Start() {
+	if s.root.Paths == nil {
+		return
+	}
+	for _, pathItem := range s.root.Paths.Paths {
+		s.walkParams(pathItem.Parameters)
+		s.walkOperation(pathItem.Delete)
+		s.walkOperation(pathItem.Get)
+		s.walkOperation(pathItem.Head)
+		s.walkOperation(pathItem.Options)
+		s.walkOperation(pathItem.Patch)
+		s.walkOperation(pathItem.Post)
+		s.walkOperation(pathItem.Put)
+	}
+}

vendor/modules.txt

@@ -880,6 +880,7 @@ k8s.io/apimachinery/pkg/api/apitesting/fuzzer
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/api/errors
 k8s.io/apimachinery/pkg/api/meta
+k8s.io/apimachinery/pkg/api/meta/table
 k8s.io/apimachinery/pkg/api/meta/testrestmapper
 k8s.io/apimachinery/pkg/api/resource
 k8s.io/apimachinery/pkg/api/validation
@@ -1068,6 +1069,7 @@ k8s.io/apiserver/pkg/util/flowcontrol/metrics
 k8s.io/apiserver/pkg/util/flowcontrol/request
 k8s.io/apiserver/pkg/util/flushwriter
 k8s.io/apiserver/pkg/util/openapi
+k8s.io/apiserver/pkg/util/proxy
 k8s.io/apiserver/pkg/util/shufflesharding
 k8s.io/apiserver/pkg/util/webhook
 k8s.io/apiserver/pkg/util/wsstream
@@ -1506,16 +1508,37 @@ k8s.io/kms/apis/v2alpha1
 # k8s.io/kube-aggregator v0.26.2
 ## explicit; go 1.19
 k8s.io/kube-aggregator/pkg/apis/apiregistration
+k8s.io/kube-aggregator/pkg/apis/apiregistration/install
 k8s.io/kube-aggregator/pkg/apis/apiregistration/v1
 k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper
 k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1
+k8s.io/kube-aggregator/pkg/apis/apiregistration/validation
+k8s.io/kube-aggregator/pkg/apiserver
+k8s.io/kube-aggregator/pkg/apiserver/scheme
 k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset
 k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme
 k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1
 k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1beta1
+k8s.io/kube-aggregator/pkg/client/informers/externalversions
+k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration
+k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1
+k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1beta1
+k8s.io/kube-aggregator/pkg/client/informers/externalversions/internalinterfaces
+k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1
+k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1beta1
+k8s.io/kube-aggregator/pkg/controllers
+k8s.io/kube-aggregator/pkg/controllers/openapi
+k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator
+k8s.io/kube-aggregator/pkg/controllers/openapiv3
+k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator
+k8s.io/kube-aggregator/pkg/controllers/status
+k8s.io/kube-aggregator/pkg/registry/apiservice
+k8s.io/kube-aggregator/pkg/registry/apiservice/etcd
+k8s.io/kube-aggregator/pkg/registry/apiservice/rest
 # k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
 ## explicit; go 1.18
 k8s.io/kube-openapi/cmd/openapi-gen/args
+k8s.io/kube-openapi/pkg/aggregator
 k8s.io/kube-openapi/pkg/builder
 k8s.io/kube-openapi/pkg/builder3
 k8s.io/kube-openapi/pkg/builder3/util