refactor(*): move getKubeClient to utils/kubernetes

(cherry picked from commit b9f636d2ef)

Signed-off-by: qianlei.qianl <qianlei.qianl@bytedance.com>

refactor: move logic to create client to utils/kubernetes pkg
- expose `CreateKubeClient` as public function
- make `GetKubeConfig` into a private `getKubeConfig` function (can be exposed as a public function in the future if needed)
Signed-off-by: vadasambar <surajrbanakar@gmail.com>

fix: CI failing because cloudproviders were not updated to use new autoscaling option fields
Signed-off-by: vadasambar <surajrbanakar@gmail.com>

refactor: define errors as constants
Signed-off-by: vadasambar <surajrbanakar@gmail.com>

refactor: pass kube client options by value
Signed-off-by: vadasambar <surajrbanakar@gmail.com>

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go

@@ -151,7 +151,7 @@ func newProvider(
 func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider {
 	managementKubeconfig := opts.CloudConfig
 	if managementKubeconfig == "" && !opts.ClusterAPICloudConfigAuthoritative {
-		managementKubeconfig = opts.KubeConfigPath
+		managementKubeconfig = opts.KubeClientOpts.KubeConfigPath
 	}
 
 	managementConfig, err := clientcmd.BuildConfigFromFlags("", managementKubeconfig)
@@ -159,7 +159,7 @@ func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupD
 		klog.Fatalf("cannot build management cluster config: %v", err)
 	}
 
-	workloadKubeconfig := opts.KubeConfigPath
+	workloadKubeconfig := opts.KubeClientOpts.KubeConfigPath
 
 	workloadConfig, err := clientcmd.BuildConfigFromFlags("", workloadKubeconfig)
 	if err != nil {

cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go

@@ -177,8 +177,8 @@ func newKamateraCloudProvider(config io.Reader, rl *cloudprovider.ResourceLimite
 }
 
 func getKubeConfig(opts config.AutoscalingOptions) *rest.Config {
-	klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath)
+	klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeClientOpts.KubeConfigPath)
-	kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath)
+	kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeClientOpts.KubeConfigPath)
 	if err != nil {
 		klog.Fatalf("Failed to build kubeConfig: %v", err)
 	}

cluster-autoscaler/cloudprovider/oci/instancepools/oci_cloud_provider.go

@@ -154,8 +154,8 @@ func BuildOCI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover
 }
 
 func getKubeConfig(opts config.AutoscalingOptions) *rest.Config {
-	klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath)
+	klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeClientOpts.KubeConfigPath)
-	kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath)
+	kubeConfig, err := clientcmd.BuildConfigFromFlags("", opts.KubeClientOpts.KubeConfigPath)
 	if err != nil {
 		klog.Fatalf("Failed to build kubeConfig: %v", err)
 	}

cluster-autoscaler/config/autoscaling_options.go

@@ -221,8 +221,8 @@ type AutoscalingOptions struct {
 	AWSUseStaticInstanceList bool
 	// GCEOptions contain autoscaling options specific to GCE cloud provider.
 	GCEOptions GCEOptions
-	// Path to kube configuration if available
+	// KubeClientOpts specify options for kube client
-	KubeConfigPath string
+	KubeClientOpts KubeClientOptions
 	// ClusterAPICloudConfigAuthoritative tells the Cluster API provider to treat the CloudConfig option as authoritative and
 	// not use KubeConfigPath as a fallback when it is not provided.
 	ClusterAPICloudConfigAuthoritative bool
@@ -274,3 +274,13 @@ type AutoscalingOptions struct {
 	// based on the latency between the CA and the api-server
 	DynamicNodeDeleteDelayAfterTaintEnabled bool
 }
+
+// KubeClientOptions specify options for kube client
+type KubeClientOptions struct {
+	// Master specifies master location.
+	Master string
+	// Path to kube configuration if available
+	KubeConfigPath string
+	// APIContentType specifies type of requests sent to APIServer.
+	APIContentType string
+}

cluster-autoscaler/main.go

@@ -21,7 +21,6 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"strconv"
@@ -61,9 +60,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/utils/units"
 	"k8s.io/autoscaler/cluster-autoscaler/version"
 	"k8s.io/client-go/informers"
-	kube_client "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	kube_flag "k8s.io/component-base/cli/flag"
@@ -353,9 +350,13 @@ func createAutoscalingOptions() config.AutoscalingOptions {
 		StatusTaints:                     *statusTaintsFlag,
 		BalancingExtraIgnoredLabels:      *balancingIgnoreLabelsFlag,
 		BalancingLabels:                  *balancingLabelsFlag,
-		KubeConfigPath:                   *kubeConfigFile,
+		KubeClientOpts: config.KubeClientOptions{
-		NodeDeletionDelayTimeout:         *nodeDeletionDelayTimeout,
+			Master:         *kubernetes,
-		AWSUseStaticInstanceList:         *awsUseStaticInstanceList,
+			KubeConfigPath: *kubeConfigFile,
+			APIContentType: *kubeAPIContentType,
+		},
+		NodeDeletionDelayTimeout: *nodeDeletionDelayTimeout,
+		AWSUseStaticInstanceList: *awsUseStaticInstanceList,
 		GCEOptions: config.GCEOptions{
 			ConcurrentRefreshes:             *concurrentGceRefreshes,
 			MigInstancesMinRefreshWaitTime:  *gceMigInstancesMinRefreshWaitTime,
@@ -391,35 +392,6 @@ func createAutoscalingOptions() config.AutoscalingOptions {
 	}
 }
 
-func getKubeConfig() *rest.Config {
-	if *kubeConfigFile != "" {
-		klog.V(1).Infof("Using kubeconfig file: %s", *kubeConfigFile)
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigFile)
-		if err != nil {
-			klog.Fatalf("Failed to build config: %v", err)
-		}
-		return config
-	}
-	url, err := url.Parse(*kubernetes)
-	if err != nil {
-		klog.Fatalf("Failed to parse Kubernetes url: %v", err)
-	}
-
-	kubeConfig, err := config.GetKubeClientConfig(url)
-	if err != nil {
-		klog.Fatalf("Failed to build Kubernetes client configuration: %v", err)
-	}
-
-	kubeConfig.ContentType = *kubeAPIContentType
-
-	return kubeConfig
-}
-
-func createKubeClient(kubeConfig *rest.Config) kube_client.Interface {
-	return kube_client.NewForConfigOrDie(kubeConfig)
-}
-
 func registerSignalHandlers(autoscaler core.Autoscaler) {
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGQUIT)
@@ -439,7 +411,7 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter
 	// Create basic config from flags.
 	autoscalingOptions := createAutoscalingOptions()
 
-	kubeClient := createKubeClient(getKubeConfig())
+	kubeClient := kube_util.CreateKubeClient(autoscalingOptions.KubeClientOpts)
 
 	// Informer transform to trim ManagedFields for memory efficiency.
 	trim := func(obj interface{}) (interface{}, error) {
@@ -618,7 +590,7 @@ func main() {
 			klog.Fatalf("Unable to get hostname: %v", err)
 		}
 
-		kubeClient := createKubeClient(getKubeConfig())
+		kubeClient := kube_util.CreateKubeClient(createAutoscalingOptions().KubeClientOpts)
 
 		// Validate that the client is ok.
 		_, err = kubeClient.CoreV1().Nodes().List(ctx.TODO(), metav1.ListOptions{})

cluster-autoscaler/utils/kubernetes/client.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"net/url"
+
+	"k8s.io/autoscaler/cluster-autoscaler/config"
+
+	kube_client "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+const (
+	failedToBuildConfigErr       = "Failed to build config"
+	failedToParseK8sUrlErr       = "Failed to parse Kubernetes url"
+	failedToBuildClientConfigErr = "Failed to build Kubernetes client configuration"
+)
+
+// CreateKubeClient creates kube client based on AutoscalingOptions.KubeClientOptions
+func CreateKubeClient(opts config.KubeClientOptions) kube_client.Interface {
+	return kube_client.NewForConfigOrDie(getKubeConfig(opts))
+}
+
+// getKubeConfig returns the rest config from AutoscalingOptions.KubeClientOptions.
+func getKubeConfig(opts config.KubeClientOptions) *rest.Config {
+	var kubeConfig *rest.Config
+	var err error
+
+	if opts.KubeConfigPath != "" {
+		klog.V(1).Infof("Using kubeconfig file: %s", opts.KubeConfigPath)
+		// use the current context in kubeconfig
+		kubeConfig, err = clientcmd.BuildConfigFromFlags("", opts.KubeConfigPath)
+		if err != nil {
+			klog.Fatalf("%v: %v", failedToBuildConfigErr, err)
+		}
+	} else {
+		url, err := url.Parse(opts.Master)
+		if err != nil {
+			klog.Fatalf("%v: %v", failedToParseK8sUrlErr, err)
+		}
+
+		kubeConfig, err = config.GetKubeClientConfig(url)
+		if err != nil {
+			klog.Fatalf("%v: %v", failedToBuildClientConfigErr, err)
+		}
+	}
+
+	kubeConfig.ContentType = opts.APIContentType
+
+	return kubeConfig
+}