From ae18f05a61dd216f1808a7678dc71ec2147ef827 Mon Sep 17 00:00:00 2001 From: "qianlei.qianl" Date: Sun, 8 Oct 2023 21:02:56 +0800 Subject: [PATCH] refactor(*): move getKubeClient to utils/kubernetes (cherry picked from commit b9f636d2efba48689fc0bb23361e5959c2177269) Signed-off-by: qianlei.qianl refactor: move logic to create client to utils/kubernetes pkg - expose `CreateKubeClient` as public function - make `GetKubeConfig` into a private `getKubeConfig` function (can be exposed as a public function in the future if needed) Signed-off-by: vadasambar fix: CI failing because cloudproviders were not updated to use new autoscaling option fields Signed-off-by: vadasambar refactor: define errors as constants Signed-off-by: vadasambar refactor: pass kube client options by value Signed-off-by: vadasambar --- .../clusterapi/clusterapi_provider.go | 4 +- .../kamatera/kamatera_cloud_provider.go | 4 +- .../oci/instancepools/oci_cloud_provider.go | 4 +- .../config/autoscaling_options.go | 14 +++- cluster-autoscaler/main.go | 46 +++---------- cluster-autoscaler/utils/kubernetes/client.go | 68 +++++++++++++++++++ 6 files changed, 95 insertions(+), 45 deletions(-) create mode 100644 cluster-autoscaler/utils/kubernetes/client.go diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go index e3c5fc1c08..cd8555fdd0 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go @@ -151,7 +151,7 @@ func newProvider( func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { managementKubeconfig := opts.CloudConfig if managementKubeconfig == "" && !opts.ClusterAPICloudConfigAuthoritative { - managementKubeconfig = opts.KubeConfigPath + managementKubeconfig = opts.KubeClientOpts.KubeConfigPath } managementConfig, err := clientcmd.BuildConfigFromFlags("", managementKubeconfig) @@ -159,7 +159,7 @@ func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupD klog.Fatalf("cannot build management cluster config: %v", err) } - workloadKubeconfig := opts.KubeConfigPath + workloadKubeconfig := opts.KubeClientOpts.KubeConfigPath workloadConfig, err := clientcmd.BuildConfigFromFlags("", workloadKubeconfig) if err != nil { diff --git a/cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go b/cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go index 4006b33fa7..ed8af5cc0f 100644 --- a/cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go @@ -177,8 +177,8 @@ func newKamateraCloudProvider(config io.Reader, rl *cloudprovider.ResourceLimite } func getKubeConfig(opts config.AutoscalingOptions) *rest.Config { - klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath) - kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath) + klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeClientOpts.KubeConfigPath) + kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeClientOpts.KubeConfigPath) if err != nil { klog.Fatalf("Failed to build kubeConfig: %v", err) } diff --git a/cluster-autoscaler/cloudprovider/oci/instancepools/oci_cloud_provider.go b/cluster-autoscaler/cloudprovider/oci/instancepools/oci_cloud_provider.go index 170828b7b7..3ddc40e160 100644 --- a/cluster-autoscaler/cloudprovider/oci/instancepools/oci_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/oci/instancepools/oci_cloud_provider.go @@ -154,8 +154,8 @@ func BuildOCI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover } func getKubeConfig(opts config.AutoscalingOptions) *rest.Config { - klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath) - kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath) + klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeClientOpts.KubeConfigPath) + kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeClientOpts.KubeConfigPath) if err != nil { klog.Fatalf("Failed to build kubeConfig: %v", err) } diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index 9e008a11a6..eaf6f3049f 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -221,8 +221,8 @@ type AutoscalingOptions struct { AWSUseStaticInstanceList bool // GCEOptions contain autoscaling options specific to GCE cloud provider. GCEOptions GCEOptions - // Path to kube configuration if available - KubeConfigPath string + // KubeClientOpts specify options for kube client + KubeClientOpts KubeClientOptions // ClusterAPICloudConfigAuthoritative tells the Cluster API provider to treat the CloudConfig option as authoritative and // not use KubeConfigPath as a fallback when it is not provided. ClusterAPICloudConfigAuthoritative bool @@ -274,3 +274,13 @@ type AutoscalingOptions struct { // based on the latency between the CA and the api-server DynamicNodeDeleteDelayAfterTaintEnabled bool } + +// KubeClientOptions specify options for kube client +type KubeClientOptions struct { + // Master specifies master location. + Master string + // Path to kube configuration if available + KubeConfigPath string + // APIContentType specifies type of requests sent to APIServer. + APIContentType string +} diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 6e5de43b5c..f35230f897 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -21,7 +21,6 @@ import ( "flag" "fmt" "net/http" - "net/url" "os" "os/signal" "strconv" @@ -61,9 +60,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/units" "k8s.io/autoscaler/cluster-autoscaler/version" "k8s.io/client-go/informers" - kube_client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" kube_flag "k8s.io/component-base/cli/flag" @@ -353,9 +350,13 @@ func createAutoscalingOptions() config.AutoscalingOptions { StatusTaints: *statusTaintsFlag, BalancingExtraIgnoredLabels: *balancingIgnoreLabelsFlag, BalancingLabels: *balancingLabelsFlag, - KubeConfigPath: *kubeConfigFile, - NodeDeletionDelayTimeout: *nodeDeletionDelayTimeout, - AWSUseStaticInstanceList: *awsUseStaticInstanceList, + KubeClientOpts: config.KubeClientOptions{ + Master: *kubernetes, + KubeConfigPath: *kubeConfigFile, + APIContentType: *kubeAPIContentType, + }, + NodeDeletionDelayTimeout: *nodeDeletionDelayTimeout, + AWSUseStaticInstanceList: *awsUseStaticInstanceList, GCEOptions: config.GCEOptions{ ConcurrentRefreshes: *concurrentGceRefreshes, MigInstancesMinRefreshWaitTime: *gceMigInstancesMinRefreshWaitTime, @@ -391,35 +392,6 @@ func createAutoscalingOptions() config.AutoscalingOptions { } } -func getKubeConfig() *rest.Config { - if *kubeConfigFile != "" { - klog.V(1).Infof("Using kubeconfig file: %s", *kubeConfigFile) - // use the current context in kubeconfig - config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigFile) - if err != nil { - klog.Fatalf("Failed to build config: %v", err) - } - return config - } - url, err := url.Parse(*kubernetes) - if err != nil { - klog.Fatalf("Failed to parse Kubernetes url: %v", err) - } - - kubeConfig, err := config.GetKubeClientConfig(url) - if err != nil { - klog.Fatalf("Failed to build Kubernetes client configuration: %v", err) - } - - kubeConfig.ContentType = *kubeAPIContentType - - return kubeConfig -} - -func createKubeClient(kubeConfig *rest.Config) kube_client.Interface { - return kube_client.NewForConfigOrDie(kubeConfig) -} - func registerSignalHandlers(autoscaler core.Autoscaler) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGQUIT) @@ -439,7 +411,7 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter // Create basic config from flags. autoscalingOptions := createAutoscalingOptions() - kubeClient := createKubeClient(getKubeConfig()) + kubeClient := kube_util.CreateKubeClient(autoscalingOptions.KubeClientOpts) // Informer transform to trim ManagedFields for memory efficiency. trim := func(obj interface{}) (interface{}, error) { @@ -618,7 +590,7 @@ func main() { klog.Fatalf("Unable to get hostname: %v", err) } - kubeClient := createKubeClient(getKubeConfig()) + kubeClient := kube_util.CreateKubeClient(createAutoscalingOptions().KubeClientOpts) // Validate that the client is ok. _, err = kubeClient.CoreV1().Nodes().List(ctx.TODO(), metav1.ListOptions{}) diff --git a/cluster-autoscaler/utils/kubernetes/client.go b/cluster-autoscaler/utils/kubernetes/client.go new file mode 100644 index 0000000000..f7f1dcc45f --- /dev/null +++ b/cluster-autoscaler/utils/kubernetes/client.go @@ -0,0 +1,68 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "net/url" + + "k8s.io/autoscaler/cluster-autoscaler/config" + + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" +) + +const ( + failedToBuildConfigErr = "Failed to build config" + failedToParseK8sUrlErr = "Failed to parse Kubernetes url" + failedToBuildClientConfigErr = "Failed to build Kubernetes client configuration" +) + +// CreateKubeClient creates kube client based on AutoscalingOptions.KubeClientOptions +func CreateKubeClient(opts config.KubeClientOptions) kube_client.Interface { + return kube_client.NewForConfigOrDie(getKubeConfig(opts)) +} + +// getKubeConfig returns the rest config from AutoscalingOptions.KubeClientOptions. +func getKubeConfig(opts config.KubeClientOptions) *rest.Config { + var kubeConfig *rest.Config + var err error + + if opts.KubeConfigPath != "" { + klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath) + // use the current context in kubeconfig + kubeConfig, err = clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath) + if err != nil { + klog.Fatalf("%v: %v", failedToBuildConfigErr, err) + } + } else { + url, err := url.Parse(opts.Master) + if err != nil { + klog.Fatalf("%v: %v", failedToParseK8sUrlErr, err) + } + + kubeConfig, err = config.GetKubeClientConfig(url) + if err != nil { + klog.Fatalf("%v: %v", failedToBuildClientConfigErr, err) + } + } + + kubeConfig.ContentType = opts.APIContentType + + return kubeConfig +}