openyurtio
/
kruise
mirror of https://github.com/openkruise/kruise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add feature-gate for Kruise to replace the CUSTOM_RESOURCE_ENABLE env (#532) 

Signed-off-by: Siyu Wang <FillZpp.pub@gmail.com>
This commit is contained in:
Siyu Wang 2021-02-13 03:05:06 +08:00 committed by GitHub
parent faa8045ae7
commit 41018db135
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 282 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
go.mod
View File

@ -24,6 +24,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.0.0
github.com/robfig/cron v1.2.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
github.com/xyproto/simpleredis v0.0.0-20200201215242-1ff0da2967b4
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
@ -36,6 +37,7 @@ require (
k8s.io/apiserver v0.16.6
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
k8s.io/code-generator v0.16.6
k8s.io/component-base v0.16.6
k8s.io/cri-api v0.16.6
k8s.io/klog v1.0.0
k8s.io/kubernetes v1.16.6
@ -66,8 +68,8 @@ replace (
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.16.6
k8s.io/metrics => k8s.io/metrics v0.16.6
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.16.6
github.com/go-bindata/go-bindata => github.com/go-bindata/go-bindata v3.1.1+incompatible
)
replace github.com/go-bindata/go-bindata => github.com/go-bindata/go-bindata v3.1.1+incompatible
replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.2

22
main.go
View File

@ -24,6 +24,12 @@ import (
"os"
"time"
extclient "github.com/openkruise/kruise/pkg/client"
_ "github.com/openkruise/kruise/pkg/features"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"github.com/openkruise/kruise/pkg/util/fieldindex"
"github.com/openkruise/kruise/pkg/webhook"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -33,14 +39,9 @@ import (
"k8s.io/kubernetes/pkg/capabilities"
ctrl "sigs.k8s.io/controller-runtime"
extclient "github.com/openkruise/kruise/pkg/client"
"github.com/openkruise/kruise/pkg/util/fieldindex"
"github.com/openkruise/kruise/pkg/webhook"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
"github.com/openkruise/kruise/pkg/controller"
"github.com/openkruise/kruise/pkg/util/gate"
// +kubebuilder:scaffold:imports
)
@ -70,7 +71,7 @@ func main() {
var namespace string
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&healthProbeAddr, "health-probe-addr", ":8000", "The address the healthz/readyz endpoint binds to.")
flag.BoolVar(&allowPrivileged, "allow-privileged", false, "If true, allow privileged containers. It will only work if api-server is also"+
flag.BoolVar(&allowPrivileged, "allow-privileged", true, "If true, allow privileged containers. It will only work if api-server is also"+
"started with --allow-privileged=true.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", true, "Whether you need to enable leader election.")
flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kruise-system",
@ -80,9 +81,12 @@ func main() {
flag.BoolVar(&enablePprof, "enable-pprof", false, "Enable pprof for controller manager.")
flag.StringVar(&pprofAddr, "pprof-addr", ":8090", "The address the pprof binds to.")
utilfeature.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine)
klog.InitFlags(nil)
flag.Parse()
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Parse()
rand.Seed(time.Now().UnixNano())
ctrl.SetLogger(klogr.New())
if enablePprof {
go func() {
@ -98,13 +102,9 @@ func main() {
})
}
//ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
ctrl.SetLogger(klogr.New())
cfg := ctrl.GetConfigOrDie()
setRestConfig(cfg)
cfg.UserAgent = "kruise-manager"
gate.Init(cfg)
setupLog.Info("new clientset registry")
err := extclient.NewRegistry(cfg)

15
pkg/client/generic_client.go
View File

@ -2,18 +2,24 @@ package client
import (
kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned"
"k8s.io/client-go/discovery"
kubeclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
// GenericClientset defines a generic client
type GenericClientset struct {
KubeClient kubeclientset.Interface
KruiseClient kruiseclientset.Interface
DiscoveryClient discovery.DiscoveryInterface
KubeClient kubeclientset.Interface
KruiseClient kruiseclientset.Interface
}
// newForConfig creates a new Clientset for the given config.
func newForConfig(c *rest.Config) (*GenericClientset, error) {
discoveryClient, err := discovery.NewDiscoveryClientForConfig(c)
if err != nil {
return nil, err
}
kubeClient, err := kubeclientset.NewForConfig(c)
if err != nil {
return nil, err
@ -23,8 +29,9 @@ func newForConfig(c *rest.Config) (*GenericClientset, error) {
return nil, err
}
return &GenericClientset{
KubeClient: kubeClient,
KruiseClient: kruiseClient,
DiscoveryClient: discoveryClient,
KubeClient: kubeClient,
KruiseClient: kruiseClient,
}, nil
}

5
pkg/controller/advancedcronjob/advancedcronjob_controller.go
View File

@ -24,7 +24,7 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/gate"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@ -49,12 +49,13 @@ func init() {
var (
concurrentReconciles = 3
jobOwnerKey = ".metadata.controller"
controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("AdvancedCronJob")
)
// Add creates a new AdvancedCronJob Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.AdvancedCronJob{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}
return add(mgr, newReconciler(mgr))

4
pkg/controller/broadcastjob/broadcastjob_controller.go
View File

@ -26,8 +26,8 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -74,7 +74,7 @@ var (
// Add creates a new BroadcastJob Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.BroadcastJob{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}
return add(mgr, newReconciler(mgr))

7
pkg/controller/cloneset/cloneset_controller.go
View File

@ -22,8 +22,6 @@ import (
"flag"
"time"
"github.com/openkruise/kruise/pkg/util"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
kruiseclient "github.com/openkruise/kruise/pkg/client"
clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core"
@ -31,9 +29,10 @@ import (
scalecontrol "github.com/openkruise/kruise/pkg/controller/cloneset/scale"
updatecontrol "github.com/openkruise/kruise/pkg/controller/cloneset/update"
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/fieldindex"
"github.com/openkruise/kruise/pkg/util/gate"
historyutil "github.com/openkruise/kruise/pkg/util/history"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
"github.com/openkruise/kruise/pkg/util/refmanager"
@ -69,7 +68,7 @@ var (
// Add creates a new CloneSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.CloneSet{}) {
if !utildiscovery.DiscoverGVK(clonesetutils.ControllerKind) {
return nil
}
return add(mgr, newReconciler(mgr))

4
pkg/controller/daemonset/daemonset_controller.go
View File

@ -25,6 +25,7 @@ import (
"sync"
"time"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -63,7 +64,6 @@ import (
"github.com/openkruise/kruise/pkg/client/clientset/versioned/scheme"
kruiseutil "github.com/openkruise/kruise/pkg/util"
kruiseExpectations "github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
)
@ -119,7 +119,7 @@ const (
// Add creates a new DaemonSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.DaemonSet{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}

6
pkg/controller/imagepulljob/imagepulljob_controller.go
View File

@ -26,9 +26,11 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
nodeimagesutil "github.com/openkruise/kruise/pkg/util/nodeimages"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -64,7 +66,7 @@ const (
// Add creates a new ImagePullJob Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.ImagePullJob{}) {
if !utildiscovery.DiscoverGVK(controllerKind) || !utilfeature.DefaultFeatureGate.Enabled(features.ImagePulling) {
return nil
}
return add(mgr, newReconciler(mgr))

6
pkg/controller/nodeimage/nodeimage_controller.go
View File

@ -26,8 +26,10 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
kruiseclient "github.com/openkruise/kruise/pkg/client"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/gate"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
nodeimagesutil "github.com/openkruise/kruise/pkg/util/nodeimages"
"github.com/openkruise/kruise/pkg/util/requeueduration"
v1 "k8s.io/api/core/v1"
@ -71,7 +73,7 @@ const (
// Add creates a new NodeImage Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.NodeImage{}) {
if !utildiscovery.DiscoverGVK(controllerKind) || !utilfeature.DefaultFeatureGate.Enabled(features.ImagePulling) {
return nil
}
return add(mgr, newReconciler(mgr))

6
pkg/controller/sidecarset/sidecarset_controller.go
View File

@ -23,10 +23,9 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
@ -45,6 +44,7 @@ func init() {
var (
concurrentReconciles = 3
controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("SidecarSet")
)
/**
@ -55,7 +55,7 @@ var (
// Add creates a new SidecarSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.SidecarSet{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}
return add(mgr, newReconciler(mgr))

4
pkg/controller/statefulset/statefulset_controller.go
View File

@ -27,8 +27,8 @@ import (
kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned"
kruiseappslisters "github.com/openkruise/kruise/pkg/client/listers/apps/v1beta1"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
"github.com/openkruise/kruise/pkg/util/requeueduration"
@ -77,7 +77,7 @@ var (
// Add creates a new StatefulSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1beta1.StatefulSet{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}
r, err := newReconciler(mgr)

14
pkg/controller/uniteddeployment/uniteddeployment_controller.go
View File

@ -22,6 +22,11 @@ import (
"fmt"
"reflect"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/controller/uniteddeployment/adapter"
"github.com/openkruise/kruise/pkg/util"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -34,12 +39,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/controller/uniteddeployment/adapter"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
)
func init() {
@ -48,6 +47,7 @@ func init() {
var (
concurrentReconciles = 3
controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("UnitedDeployment")
)
const (
@ -74,7 +74,7 @@ const (
// Add creates a new UnitedDeployment Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager) error {
if !gate.ResourceEnabled(&appsv1alpha1.UnitedDeployment{}) {
if !utildiscovery.DiscoverGVK(controllerKind) {
return nil
}
return add(mgr, newReconciler(mgr))

57
pkg/features/kruise_features.go Normal file
View File

@ -0,0 +1,57 @@
/*
Copyright 2021 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"os"
"strings"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/component-base/featuregate"
)
const (
// ImagePulling enables controllers for NodeImage and ImagePullJob.
ImagePulling featuregate.Feature = "ImagePulling"
// PodWebhook enables webhook for Pods creations. This is also related to SidecarSet.
PodWebhook featuregate.Feature = "PodWebhook"
)
var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
PodWebhook: {Default: true, PreRelease: featuregate.Beta},
ImagePulling: {Default: true, PreRelease: featuregate.Beta},
}
func init() {
compatibleEnv()
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultFeatureGates))
}
// Make it compatible with the old CUSTOM_RESOURCE_ENABLE gate in env.
func compatibleEnv() {
str := strings.TrimSpace(os.Getenv("CUSTOM_RESOURCE_ENABLE"))
if len(str) == 0 {
return
}
limits := sets.NewString(strings.Split(str, ",")...)
if !limits.Has("SidecarSet") {
defaultFeatureGates[PodWebhook] = featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}
}
}

81
pkg/util/discovery/discovery.go Normal file
View File

@ -0,0 +1,81 @@
/*
Copyright 2021 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package discovery
import (
"github.com/openkruise/kruise/apis"
"github.com/openkruise/kruise/pkg/client"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/util/retry"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
var (
internalScheme = runtime.NewScheme()
isNotNotFound = func(err error) bool { return !errors.IsNotFound(err) }
)
func init() {
_ = apis.AddToScheme(internalScheme)
}
func DiscoverGVK(gvk schema.GroupVersionKind) bool {
genericClient := client.GetGenericClient()
if genericClient == nil {
return true
}
discoveryClient := genericClient.DiscoveryClient
var resourceList *metav1.APIResourceList
err := retry.OnError(retry.DefaultBackoff, isNotNotFound, func() error {
var err error
resourceList, err = discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
if err != nil && !errors.IsNotFound(err) {
klog.Infof("Failed to get groupVersionKind %v: %v", gvk, err)
}
return err
})
if err != nil {
if errors.IsNotFound(err) {
klog.Infof("Not found groupVersionKind %v: %v", gvk, err)
return false
}
// This might be caused by abnormal apiserver or etcd, ignore it
return true
}
for _, r := range resourceList.APIResources {
if r.Kind == gvk.Kind {
return true
}
}
return false
}
func DiscoverObject(obj runtime.Object) bool {
gvk, err := apiutil.GVKForObject(obj, internalScheme)
if err != nil {
klog.Warningf("Not recognized object %T in scheme: %v", obj, err)
return false
}
return DiscoverGVK(gvk)
}

33
pkg/util/feature/feature_gate.go Normal file
View File

@ -0,0 +1,33 @@
/*
Copyright 2021 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package feature
import (
"k8s.io/component-base/featuregate"
)
var (
// DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate.
// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
// Tests that need to modify feature gates for the duration of their test should use:
// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()
DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
// DefaultFeatureGate is a shared global FeatureGate.
// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.
DefaultFeatureGate featuregate.FeatureGate = DefaultMutableFeatureGate
)

4
pkg/util/fieldindex/register.go
View File

@ -20,7 +20,7 @@ import (
"sync"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util/gate"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -72,7 +72,7 @@ func RegisterFieldIndexes(c cache.Cache) error {
return
}
// broadcastjob owner
if gate.ResourceEnabled(&appsv1alpha1.BroadcastJob{}) {
if utildiscovery.DiscoverObject(&appsv1alpha1.BroadcastJob{}) {
if err = indexBroadcastCronJob(c); err != nil {
return
}

112
pkg/util/gate/custom_resource_gate.go
View File

@ -1,112 +0,0 @@
/*
Copyright 2019 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gate
import (
"os"
"strings"
"github.com/openkruise/kruise/apis"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
const (
envCustomResourceEnable = "CUSTOM_RESOURCE_ENABLE"
)
var (
internalScheme = runtime.NewScheme()
discoveryClient discovery.DiscoveryInterface
isNotNotFound = func(err error) bool { return !errors.IsNotFound(err) }
)
func init() {
_ = apis.AddToScheme(internalScheme)
}
func Init(cfg *rest.Config) {
discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(cfg)
}
// ResourceEnabled help runnable check if the custom resource is valid and enabled
// 1. If this CRD is not found from kube-apiserver, it is invalid.
// 2. If 'CUSTOM_RESOURCE_ENABLE' env is not empty and this CRD kind is not in ${CUSTOM_RESOURCE_ENABLE}.
func ResourceEnabled(obj runtime.Object) bool {
gvk, err := apiutil.GVKForObject(obj, internalScheme)
if err != nil {
klog.Warningf("custom resource gate not recognized object %T in scheme: %v", obj, err)
return false
}
return discoveryEnabled(gvk) && envEnabled(gvk)
}
func discoveryEnabled(gvk schema.GroupVersionKind) bool {
if discoveryClient == nil {
return true
}
var resourceList *metav1.APIResourceList
err := retry.OnError(retry.DefaultBackoff, isNotNotFound, func() error {
var err error
resourceList, err = discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
if err != nil && !errors.IsNotFound(err) {
klog.Infof("custom resource gate failed to get groupVersionKind %v in discovery: %v", gvk, err)
}
return err
})
if err != nil {
if errors.IsNotFound(err) {
klog.Infof("custom resource gate not found groupVersionKind %v in discovery: %v", gvk, err)
return false
}
// This might be caused by abnormal apiserver or etcd, ignore the discovery and just use envEnable
return true
}
for _, r := range resourceList.APIResources {
if r.Kind == gvk.Kind {
return true
}
}
return false
}
func envEnabled(gvk schema.GroupVersionKind) bool {
limits := strings.TrimSpace(os.Getenv(envCustomResourceEnable))
if len(limits) == 0 {
// all enabled by default
return true
}
if !sets.NewString(strings.Split(limits, ",")...).Has(gvk.Kind) {
klog.Warningf("custom resource gate not found groupVersionKind %v in CUSTOM_RESOURCE_ENABLE: %v", gvk, limits)
return false
}
return true
}

16
pkg/webhook/add_pod.go
View File

@ -18,9 +18,23 @@ package webhook
import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/features"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"github.com/openkruise/kruise/pkg/webhook/pod/mutating"
)
func init() {
addHandlersWithGate(mutating.HandlerMap, &appsv1alpha1.SidecarSet{})
addHandlersWithGate(mutating.HandlerMap, func() (enabled bool) {
if !utilfeature.DefaultFeatureGate.Enabled(features.PodWebhook) {
return false
}
// Currently, if SidecarSet is not installed, we can also disable pod webhook.
if !utildiscovery.DiscoverObject(&appsv1alpha1.SidecarSet{}) {
return false
}
return true
})
}

15
pkg/webhook/advancedcronjob/validating/advancedcronjob_create_update_handler.go
View File

@ -22,24 +22,19 @@ import (
"net/http"
"regexp"
"github.com/robfig/cron"
apiequality "k8s.io/apimachinery/pkg/api/equality"
batchv1beta1 "k8s.io/api/batch/v1beta1"
corevalidation "k8s.io/kubernetes/pkg/apis/core/validation"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/robfig/cron"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
batchv1beta1 "k8s.io/api/batch/v1beta1"
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
genericvalidation "k8s.io/apimachinery/pkg/api/validation"
validationutil "k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/kubernetes/pkg/apis/core"
corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
corevalidation "k8s.io/kubernetes/pkg/apis/core/validation"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

4
pkg/webhook/daemonset/validating/daemonset_create_update_handler.go
View File

@ -21,17 +21,15 @@ import (
"fmt"
"net/http"
appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/webhook/util/convertor"
genericvalidation "k8s.io/apimachinery/pkg/api/validation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/klog"
appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation"
corevalidation "k8s.io/kubernetes/pkg/apis/core/validation"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

8
pkg/webhook/imagepulljob/validating/imagepulljob_create_update_handler.go
View File

@ -21,9 +21,10 @@ import (
"fmt"
"net/http"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
"github.com/openkruise/kruise/pkg/features"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
@ -44,6 +45,9 @@ func (h *ImagePullJobCreateUpdateHandler) Handle(ctx context.Context, req admiss
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
if !utilfeature.DefaultFeatureGate.Enabled(features.ImagePulling) {
return admission.Errored(http.StatusForbidden, fmt.Errorf("feature-gate %s is not enabled", features.ImagePulling))
}
if err := validate(obj); err != nil {
klog.Warningf("Error validate ImagePullJob %s/%s: %v", obj.Namespace, obj.Name, err)

8
pkg/webhook/nodeimage/validating/nodeimage_create_update_handler.go
View File

@ -21,9 +21,10 @@ import (
"fmt"
"net/http"
"k8s.io/apimachinery/pkg/util/sets"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/features"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
@ -48,6 +49,9 @@ func (h *NodeImageCreateUpdateHandler) Handle(ctx context.Context, req admission
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
if !utilfeature.DefaultFeatureGate.Enabled(features.ImagePulling) {
return admission.Errored(http.StatusForbidden, fmt.Errorf("feature-gate %s is not enabled", features.ImagePulling))
}
if err := validate(obj); err != nil {
klog.Warningf("Error validate NodeImage %s: %v", obj.Name, err)

16
pkg/webhook/server.go
View File

@ -20,11 +20,9 @@ import (
"fmt"
"time"
"github.com/openkruise/kruise/pkg/util/gate"
webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
webhookcontroller "github.com/openkruise/kruise/pkg/webhook/util/controller"
"github.com/openkruise/kruise/pkg/webhook/util/health"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
@ -34,10 +32,12 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook/conversion"
)
type GateFunc func() (enabled bool)
var (
// HandlerMap contains all admission webhook handlers.
HandlerMap = map[string]admission.Handler{}
handlerGates = map[string]runtime.Object{}
handlerGates = map[string]GateFunc{}
Checker = health.Checker
)
@ -46,7 +46,7 @@ func addHandlers(m map[string]admission.Handler) {
addHandlersWithGate(m, nil)
}
func addHandlersWithGate(m map[string]admission.Handler, gateObj runtime.Object) {
func addHandlersWithGate(m map[string]admission.Handler, fn GateFunc) {
for path, handler := range m {
if len(path) == 0 {
klog.Warningf("Skip handler with empty path.")
@ -60,8 +60,8 @@ func addHandlersWithGate(m map[string]admission.Handler, gateObj runtime.Object)
klog.V(1).Infof("conflicting webhook builder path %v in handler map", path)
}
HandlerMap[path] = handler
if gateObj != nil {
handlerGates[path] = gateObj
if fn != nil {
handlerGates[path] = fn
}
}
}
@ -69,8 +69,8 @@ func addHandlersWithGate(m map[string]admission.Handler, gateObj runtime.Object)
func filterActiveHandlers() {
disablePaths := sets.NewString()
for path := range HandlerMap {
if obj, ok := handlerGates[path]; ok {
if !gate.ResourceEnabled(obj) {
if fn, ok := handlerGates[path]; ok {
if !fn() {
disablePaths.Insert(path)
}
}

7
pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go
View File

@ -24,8 +24,9 @@ import (
"regexp"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
v1 "k8s.io/api/core/v1"
genericvalidation "k8s.io/apimachinery/pkg/api/validation"
@ -335,6 +336,10 @@ func (h *SidecarSetCreateUpdateHandler) Handle(ctx context.Context, req admissio
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
if !utilfeature.DefaultFeatureGate.Enabled(features.PodWebhook) {
return admission.Errored(http.StatusForbidden, fmt.Errorf("feature-gate %s is not enabled", features.PodWebhook))
}
var oldSidecarSet *appsv1alpha1.SidecarSet
//when Operation is update, decode older object
if req.AdmissionRequest.Operation == admissionv1beta1.Update {

1
pkg/webhook/statefulset/validating/statefulset_create_update_handler.go
View File

@ -22,7 +22,6 @@ import (
"net/http"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

4
pkg/webhook/util/configuration/configuration.go
View File

@ -63,7 +63,7 @@ func Ensure(c client.Client, handlers map[string]admission.Handler, caBundle []b
return err
}
if _, ok := handlers[path]; !ok {
klog.Warningf("Find path %s not in handlers %v", path, handlers)
klog.Warningf("Ignore webhook for %s in configuration", path)
continue
}
if wh.ClientConfig.Service != nil {
@ -86,7 +86,7 @@ func Ensure(c client.Client, handlers map[string]admission.Handler, caBundle []b
return err
}
if _, ok := handlers[path]; !ok {
klog.Warningf("Find path %s not in handlers %v", path, handlers)
klog.Warningf("Ignore webhook for %s in configuration", path)
continue
}
if wh.ClientConfig.Service != nil {

8
pkg/webhook/util/writer/atomic/atomic_writer.go
View File

@ -152,7 +152,7 @@ func (w *Writer) Write(payload map[string]FileProjection) error {
klog.Error(err, "unable to determine whether payload should be written to disk")
return err
} else if !should && len(pathsToRemove) == 0 {
klog.V(1).Info("no update required for target directory", "directory", w.targetDir)
klog.V(6).Info("no update required for target directory", "directory", w.targetDir)
return nil
} else {
klog.V(1).Info("write required for target directory", "directory", w.targetDir)
@ -327,7 +327,6 @@ func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir strin
} else if err != nil {
return nil, err
}
klog.V(1).Info("current paths", "target directory", w.targetDir, "paths", paths.List())
newPaths := sets.NewString()
for file := range payload {
@ -339,10 +338,11 @@ func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir strin
subPath = strings.TrimSuffix(subPath, string(os.PathSeparator))
}
}
klog.V(1).Info("new paths", "target directory", w.targetDir, "paths", newPaths.List())
result := paths.Difference(newPaths)
klog.V(1).Info("paths to remove", "target directory", w.targetDir, "paths", result)
if len(result) > 0 {
klog.V(1).Info("paths to remove", "target directory", w.targetDir, "paths", result)
}
return result, nil
}