diff --git a/pkg/karmadactl/addons/metricsadapter/manifests.go b/pkg/karmadactl/addons/metricsadapter/manifests.go index 7c14f15a7..beed0d84b 100644 --- a/pkg/karmadactl/addons/metricsadapter/manifests.go +++ b/pkg/karmadactl/addons/metricsadapter/manifests.go @@ -102,7 +102,7 @@ spec: namespace: {{ .Namespace }} group: {{ .Group }} version: {{ .Version }} - insecureSkipTLSVerify: true + caBundle: {{ .CABundle }} groupPriorityMinimum: 100 versionPriority: 200 ` @@ -140,6 +140,7 @@ type AAApiServiceReplace struct { Namespace string Group string Version string + CABundle string } // AAServiceReplace is a struct to help to concrete diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go index 1e772a1a8..c00a616ea 100644 --- a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go @@ -2,6 +2,7 @@ package metricsadapter import ( "context" + "encoding/base64" "fmt" "strings" "time" @@ -19,6 +20,7 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" + "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) @@ -169,6 +171,12 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO return fmt.Errorf("error when parsing karmada metrics adapter AA service template :%v", err) } + caCertName := fmt.Sprintf("%s.crt", options.CaCertAndKeyName) + karmadaCerts, err := opts.KubeClientSet.CoreV1().Secrets(opts.Namespace).Get(context.TODO(), options.KarmadaCertsName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error when getting Secret %s/%s, which is used to fetch CaCert for building APISevice: %+v", opts.Namespace, options.KarmadaCertsName, err) + } + aaService := &corev1.Service{} if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaServiceBytes, aaService); err != nil { return fmt.Errorf("decode karmada metrics adapter AA service error: %v", err) @@ -184,6 +192,7 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO Namespace: opts.Namespace, Group: gv[1], Version: gv[0], + CABundle: base64.StdEncoding.EncodeToString(karmadaCerts.Data[caCertName]), }) if err != nil { return fmt.Errorf("error when parsing karmada metrics adapter AA apiservice template :%v", err) diff --git a/pkg/karmadactl/addons/search/manifests.go b/pkg/karmadactl/addons/search/manifests.go index 818f97817..264e22714 100644 --- a/pkg/karmadactl/addons/search/manifests.go +++ b/pkg/karmadactl/addons/search/manifests.go @@ -98,7 +98,7 @@ metadata: app: karmada-search apiserver: "true" spec: - insecureSkipTLSVerify: true + caBundle: {{ .CABundle }} group: search.karmada.io groupPriorityMinimum: 2000 service: @@ -141,6 +141,7 @@ type ServiceReplace struct { type AAApiServiceReplace struct { Name string Namespace string + CABundle string } // AAServiceReplace is a struct to help to concrete diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 9d488cc55..df679bc79 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -2,6 +2,7 @@ package search import ( "context" + "encoding/base64" "fmt" "strings" "time" @@ -20,6 +21,7 @@ import ( addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" + "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" ) @@ -182,6 +184,12 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO return fmt.Errorf("error when parsing karmada search AA service template :%v", err) } + caCertName := fmt.Sprintf("%s.crt", options.CaCertAndKeyName) + karmadaCerts, err := opts.KubeClientSet.CoreV1().Secrets(opts.Namespace).Get(context.TODO(), options.KarmadaCertsName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error when getting Secret %s/%s, which is used to fetch CaCert for building APISevice: %+v", opts.Namespace, options.KarmadaCertsName, err) + } + aaService := &corev1.Service{} if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaServiceBytes, aaService); err != nil { return fmt.Errorf("decode karmada search AA service error: %v", err) @@ -194,6 +202,7 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO aaAPIServiceBytes, err := addonutils.ParseTemplate(karmadaSearchAAAPIService, AAApiServiceReplace{ Name: aaAPIServiceName, Namespace: opts.Namespace, + CABundle: base64.StdEncoding.EncodeToString(karmadaCerts.Data[caCertName]), }) if err != nil { return fmt.Errorf("error when parsing karmada search AA apiservice template :%v", err) diff --git a/pkg/karmadactl/cmdinit/cert/cert.go b/pkg/karmadactl/cmdinit/cert/cert.go index 60c6fb5a9..6a6c64231 100644 --- a/pkg/karmadactl/cmdinit/cert/cert.go +++ b/pkg/karmadactl/cmdinit/cert/cert.go @@ -23,6 +23,7 @@ import ( "k8s.io/kube-openapi/pkg/util/sets" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" + globaloptions "github.com/karmada-io/karmada/pkg/karmadactl/options" ) const ( @@ -249,7 +250,7 @@ func GenCerts(pkiPath string, etcdServerCertCfg, etcdClientCertCfg, karmadaCertC if err != nil { return err } - if err = WriteCertAndKey(pkiPath, options.CaCertAndKeyName, caCert, caKey); err != nil { + if err = WriteCertAndKey(pkiPath, globaloptions.CaCertAndKeyName, caCert, caKey); err != nil { return err } diff --git a/pkg/karmadactl/cmdinit/karmada/deploy.go b/pkg/karmadactl/cmdinit/karmada/deploy.go index 606ffe3dd..5516e4c65 100644 --- a/pkg/karmadactl/cmdinit/karmada/deploy.go +++ b/pkg/karmadactl/cmdinit/karmada/deploy.go @@ -2,6 +2,7 @@ package karmada import ( "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -100,7 +101,7 @@ func InitKarmadaResources(dir, caBase64, systemNamespace string) error { // karmada-aggregated-apiserver klog.Info("Create Service 'karmada-aggregated-apiserver' and APIService 'v1alpha1.cluster.karmada.io'.") - if err = initAggregatedAPIService(clientSet, restConfig, systemNamespace); err != nil { + if err = initAggregatedAPIService(clientSet, restConfig, systemNamespace, caBase64); err != nil { klog.Exitln(err) } @@ -260,8 +261,12 @@ func getName(str, start, end string) string { return str } -func initAggregatedAPIService(clientSet *kubernetes.Clientset, restConfig *rest.Config, systemNamespace string) error { +func initAggregatedAPIService(clientSet *kubernetes.Clientset, restConfig *rest.Config, systemNamespace, caBase64 string) error { // https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml + caBytes, err := base64.StdEncoding.DecodeString(caBase64) + if err != nil { + return fmt.Errorf("failed to decode caBase64: %+v", err) + } aaService := &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -297,9 +302,9 @@ func initAggregatedAPIService(clientSet *kubernetes.Clientset, restConfig *rest. Labels: map[string]string{"app": "karmada-aggregated-apiserver", "apiserver": "true"}, }, Spec: apiregistrationv1.APIServiceSpec{ - InsecureSkipTLSVerify: true, - Group: clusterv1alpha1.GroupName, - GroupPriorityMinimum: 2000, + CABundle: caBytes, + Group: clusterv1alpha1.GroupName, + GroupPriorityMinimum: 2000, Service: &apiregistrationv1.ServiceReference{ Name: aaService.Name, Namespace: aaService.Namespace, diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index 44cd8575e..b356c66f8 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -23,6 +23,7 @@ import ( "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" + globaloptions "github.com/karmada-io/karmada/pkg/karmadactl/options" "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" "github.com/karmada-io/karmada/pkg/version" @@ -35,7 +36,7 @@ var ( } certList = []string{ - options.CaCertAndKeyName, + globaloptions.CaCertAndKeyName, options.EtcdCaCertAndKeyName, options.EtcdServerCertAndKeyName, options.EtcdClientCertAndKeyName, @@ -358,7 +359,7 @@ func (i *CommandInitOption) prepareCRD() error { func (i *CommandInitOption) createCertsSecrets() error { // Create kubeconfig Secret karmadaServerURL := fmt.Sprintf("https://%s.%s.svc.%s:%v", karmadaAPIServerDeploymentAndServiceName, i.Namespace, i.HostClusterDomain, karmadaAPIServerContainerPort) - config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.CaCertAndKeyName)], + config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]) configBytes, err := clientcmd.Write(*config) if err != nil { @@ -386,7 +387,7 @@ func (i *CommandInitOption) createCertsSecrets() error { karmadaCert[fmt.Sprintf("%s.crt", v)] = string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", v)]) karmadaCert[fmt.Sprintf("%s.key", v)] = string(i.CertAndKeyFileData[fmt.Sprintf("%s.key", v)]) } - karmadaSecret := i.SecretFromSpec(karmadaCertsName, corev1.SecretTypeOpaque, karmadaCert) + karmadaSecret := i.SecretFromSpec(globaloptions.KarmadaCertsName, corev1.SecretTypeOpaque, karmadaCert) if err := util.CreateOrUpdateSecret(i.KubeClientSet, karmadaSecret); err != nil { return err } @@ -571,7 +572,7 @@ func (i *CommandInitOption) RunInit(parentCommand string) error { } // Create CRDs in karmada - caBase64 := base64.StdEncoding.EncodeToString(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.CaCertAndKeyName)]) + caBase64 := base64.StdEncoding.EncodeToString(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)]) if err := karmada.InitKarmadaResources(i.KarmadaDataPath, caBase64, i.Namespace); err != nil { return err } @@ -598,7 +599,7 @@ func (i *CommandInitOption) createKarmadaConfig() error { return err } if err := utils.WriteKubeConfigFromSpec(serverURL, options.UserName, options.ClusterName, i.KarmadaDataPath, options.KarmadaKubeConfigName, - i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.CaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], + i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]); err != nil { return fmt.Errorf("failed to create karmada kubeconfig file. %v", err) } diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index fe526eed9..7e54a51ff 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -12,6 +12,7 @@ import ( "k8s.io/utils/pointer" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" + globaloptions "github.com/karmada-io/karmada/pkg/karmadactl/options" ) const ( @@ -21,7 +22,6 @@ const ( // KubeConfigSecretAndMountName is the secret and volume mount name of karmada kubeconfig KubeConfigSecretAndMountName = "kubeconfig" - karmadaCertsName = "karmada-cert" karmadaCertsVolumeMountPath = "/etc/karmada/pki" kubeConfigContainerMountPath = "/etc/kubeconfig" karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver" @@ -67,7 +67,7 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { "kube-apiserver", "--allow-privileged=true", "--authorization-mode=Node,RBAC", - fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), + fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--enable-bootstrap-token-auth=true", fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), @@ -180,7 +180,7 @@ func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment }, VolumeMounts: []corev1.VolumeMount{ { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath, }, @@ -191,10 +191,10 @@ func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment }, Volumes: []corev1.Volume{ { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: karmadaCertsName, + SecretName: globaloptions.KarmadaCertsName, }, }, }, @@ -290,17 +290,17 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 "--authentication-kubeconfig=/etc/kubeconfig", "--authorization-kubeconfig=/etc/kubeconfig", "--bind-address=0.0.0.0", - fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), + fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--cluster-cidr=10.244.0.0/16", fmt.Sprintf("--cluster-name=%s", options.ClusterName), - fmt.Sprintf("--cluster-signing-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), - fmt.Sprintf("--cluster-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), + fmt.Sprintf("--cluster-signing-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), + fmt.Sprintf("--cluster-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrapproving,csrcleaner,csrsigning,clusterrole-aggregation", "--kubeconfig=/etc/kubeconfig", "--leader-elect=true", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--node-cidr-mask-size=24", - fmt.Sprintf("--root-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), + fmt.Sprintf("--root-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), fmt.Sprintf("--service-account-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName), fmt.Sprintf("--service-cluster-ip-range=%s", serviceClusterIP), "--use-service-account-credentials=true", @@ -322,7 +322,7 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 SubPath: KubeConfigSecretAndMountName, }, { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath, }, @@ -339,10 +339,10 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, }, { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: karmadaCertsName, + SecretName: globaloptions.KarmadaCertsName, }, }, }, @@ -840,7 +840,7 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D SubPath: KubeConfigSecretAndMountName, }, { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, ReadOnly: true, MountPath: karmadaCertsVolumeMountPath, }, @@ -864,10 +864,10 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D }, }, { - Name: karmadaCertsName, + Name: globaloptions.KarmadaCertsName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: karmadaCertsName, + SecretName: globaloptions.KarmadaCertsName, }, }, }, diff --git a/pkg/karmadactl/cmdinit/options/global.go b/pkg/karmadactl/cmdinit/options/global.go index 6dd61e60f..cb257cddf 100644 --- a/pkg/karmadactl/cmdinit/options/global.go +++ b/pkg/karmadactl/cmdinit/options/global.go @@ -1,8 +1,6 @@ package options const ( - // CaCertAndKeyName ca certificate key name - CaCertAndKeyName = "ca" // EtcdCaCertAndKeyName etcd ca certificate key name EtcdCaCertAndKeyName = "etcd-ca" // EtcdServerCertAndKeyName etcd server certificate key name diff --git a/pkg/karmadactl/options/global.go b/pkg/karmadactl/options/global.go index db32cb1f4..acb6c9d02 100644 --- a/pkg/karmadactl/options/global.go +++ b/pkg/karmadactl/options/global.go @@ -16,6 +16,13 @@ const DefaultHostClusterDomain = "cluster.local" // DefaultKarmadactlCommandDuration defines the default timeout for karmadactl execute const DefaultKarmadactlCommandDuration = 60 * time.Second +const ( + // KarmadaCertsName the secret name of karmada certs + KarmadaCertsName = "karmada-cert" + // CaCertAndKeyName ca certificate cert/key name in karmada certs secret + CaCertAndKeyName = "ca" +) + // DefaultConfigFlags It composes the set of values necessary for obtaining a REST client config with default values set. var DefaultConfigFlags = genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)