Merge pull request #5842 from jabellard/external_ca_cert

Support Custom API Server CA Certificate for Karmada Instance in Operator

charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml

@@ -3673,6 +3673,32 @@ spec:
                         type: string
                     type: object
                 type: object
+              customCertificate:
+                description: |-
+                  CustomCertificate specifies the configuration to customize the certificates
+                  for Karmada components or control the certificate generation process, such as
+                  the algorithm, validity period, etc.
+                  Currently, it only supports customizing the CA certificate for limited components.
+                properties:
+                  apiServerCACert:
+                    description: |-
+                      APIServerCACert references a Kubernetes secret containing the CA certificate
+                      for component karmada-apiserver.
+                      The secret must contain the following data keys:
+                      - tls.crt: The TLS certificate.
+                      - tls.key: The TLS private key.
+                      If specified, this CA will be used to issue client certificates for
+                      all components that access the APIServer as clients.
+                    properties:
+                      name:
+                        description: Name is the name of resource being referenced.
+                        type: string
+                      namespace:
+                        description: Namespace is the namespace for the resource being
+                          referenced.
+                        type: string
+                    type: object
+                type: object
               featureGates:
                 additionalProperties:
                   type: boolean

operator/config/crds/operator.karmada.io_karmadas.yaml

@@ -3673,6 +3673,32 @@ spec:
                         type: string
                     type: object
                 type: object
+              customCertificate:
+                description: |-
+                  CustomCertificate specifies the configuration to customize the certificates
+                  for Karmada components or control the certificate generation process, such as
+                  the algorithm, validity period, etc.
+                  Currently, it only supports customizing the CA certificate for limited components.
+                properties:
+                  apiServerCACert:
+                    description: |-
+                      APIServerCACert references a Kubernetes secret containing the CA certificate
+                      for component karmada-apiserver.
+                      The secret must contain the following data keys:
+                      - tls.crt: The TLS certificate.
+                      - tls.key: The TLS private key.
+                      If specified, this CA will be used to issue client certificates for
+                      all components that access the APIServer as clients.
+                    properties:
+                      name:
+                        description: Name is the name of resource being referenced.
+                        type: string
+                      namespace:
+                        description: Namespace is the namespace for the resource being
+                          referenced.
+                        type: string
+                    type: object
+                type: object
               featureGates:
                 additionalProperties:
                   type: boolean

operator/pkg/apis/operator/v1alpha1/type.go

@@ -113,6 +113,26 @@ type KarmadaSpec struct {
 	// By default, the operator will only attempt to download the tarball if it's not yet present in the local cache.
 	// +optional
 	CRDTarball *CRDTarball `json:"crdTarball,omitempty"`
+
+	// CustomCertificate specifies the configuration to customize the certificates
+	// for Karmada components or control the certificate generation process, such as
+	// the algorithm, validity period, etc.
+	// Currently, it only supports customizing the CA certificate for limited components.
+	// +optional
+	CustomCertificate *CustomCertificate `json:"customCertificate,omitempty"`
+}
+
+// CustomCertificate holds the configuration for generating the certificate.
+type CustomCertificate struct {
+	// APIServerCACert references a Kubernetes secret containing the CA certificate
+	// for component karmada-apiserver.
+	// The secret must contain the following data keys:
+	// - tls.crt: The TLS certificate.
+	// - tls.key: The TLS private key.
+	// If specified, this CA will be used to issue client certificates for
+	// all components that access the APIServer as clients.
+	// +optional
+	APIServerCACert *LocalSecretReference `json:"apiServerCACert,omitempty"`
 }
 
 // ImageRegistry represents an image registry as well as the

operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go

@@ -106,6 +106,27 @@ func (in *CommonSettings) DeepCopy() *CommonSettings {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CustomCertificate) DeepCopyInto(out *CustomCertificate) {
+	*out = *in
+	if in.APIServerCACert != nil {
+		in, out := &in.APIServerCACert, &out.APIServerCACert
+		*out = new(LocalSecretReference)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomCertificate.
+func (in *CustomCertificate) DeepCopy() *CustomCertificate {
+	if in == nil {
+		return nil
+	}
+	out := new(CustomCertificate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Etcd) DeepCopyInto(out *Etcd) {
 	*out = *in
@@ -637,6 +658,11 @@ func (in *KarmadaSpec) DeepCopyInto(out *KarmadaSpec) {
 		*out = new(CRDTarball)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.CustomCertificate != nil {
+		in, out := &in.CustomCertificate, &out.CustomCertificate
+		*out = new(CustomCertificate)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 

operator/pkg/certs/certs.go

@@ -218,6 +218,11 @@ type KarmadaCert struct {
 	key      []byte
 }
 
+// NewKarmadaCert is used to create a new Karmada cert
+func NewKarmadaCert(pairName, caName string, cert, key []byte) *KarmadaCert {
+	return &KarmadaCert{pairName: pairName, caName: caName, cert: cert, key: key}
+}
+
 // CertData returns certificate cert data.
 func (cert *KarmadaCert) CertData() []byte {
 	return cert.cert

operator/pkg/init.go

@@ -42,13 +42,14 @@ var (
 
 // InitOptions defines all the init workflow options.
 type InitOptions struct {
-	Name           string
+	Name                    string
-	Namespace      string
+	Namespace               string
-	Kubeconfig     *rest.Config
+	Kubeconfig              *rest.Config
-	KarmadaVersion string
+	KarmadaVersion          string
-	CRDTarball     operatorv1alpha1.CRDTarball
+	CRDTarball              operatorv1alpha1.CRDTarball
-	KarmadaDataDir string
+	CustomCertificateConfig operatorv1alpha1.CustomCertificate
-	Karmada        *operatorv1alpha1.Karmada
+	KarmadaDataDir          string
+	Karmada                 *operatorv1alpha1.Karmada
 }
 
 // Validate is used to validate the initOptions before creating initJob.
@@ -75,19 +76,20 @@ var _ tasks.InitData = &initData{}
 type initData struct {
 	sync.Once
 	certs.CertStore
-	name                string
+	name                    string
-	namespace           string
+	namespace               string
-	karmadaVersion      *utilversion.Version
+	karmadaVersion          *utilversion.Version
-	controlplaneConfig  *rest.Config
+	controlplaneConfig      *rest.Config
-	controlplaneAddress string
+	controlplaneAddress     string
-	remoteClient        clientset.Interface
+	remoteClient            clientset.Interface
-	karmadaClient       clientset.Interface
+	karmadaClient           clientset.Interface
-	dnsDomain           string
+	dnsDomain               string
-	CRDTarball          operatorv1alpha1.CRDTarball
+	CRDTarball              operatorv1alpha1.CRDTarball
-	karmadaDataDir      string
+	CustomCertificateConfig operatorv1alpha1.CustomCertificate
-	privateRegistry     string
+	karmadaDataDir          string
-	featureGates        map[string]bool
+	privateRegistry         string
-	components          *operatorv1alpha1.KarmadaComponents
+	featureGates            map[string]bool
+	components              *operatorv1alpha1.KarmadaComponents
 }
 
 // NewInitJob initializes a job with list of init sub-task. and build
@@ -165,18 +167,19 @@ func newRunData(opt *InitOptions) (*initData, error) {
 	}
 
 	return &initData{
-		name:                opt.Name,
+		name:                    opt.Name,
-		namespace:           opt.Namespace,
+		namespace:               opt.Namespace,
-		karmadaVersion:      version,
+		karmadaVersion:          version,
-		controlplaneAddress: address,
+		controlplaneAddress:     address,
-		remoteClient:        remoteClient,
+		remoteClient:            remoteClient,
-		CRDTarball:          opt.CRDTarball,
+		CRDTarball:              opt.CRDTarball,
-		karmadaDataDir:      opt.KarmadaDataDir,
+		CustomCertificateConfig: opt.CustomCertificateConfig,
-		privateRegistry:     privateRegistry,
+		karmadaDataDir:          opt.KarmadaDataDir,
-		components:          opt.Karmada.Spec.Components,
+		privateRegistry:         privateRegistry,
-		featureGates:        opt.Karmada.Spec.FeatureGates,
+		components:              opt.Karmada.Spec.Components,
-		dnsDomain:           *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
+		featureGates:            opt.Karmada.Spec.FeatureGates,
-		CertStore:           certs.NewCertStore(),
+		dnsDomain:               *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
+		CertStore:               certs.NewCertStore(),
 	}, nil
 }
 
@@ -226,6 +229,10 @@ func (data *initData) CrdTarball() operatorv1alpha1.CRDTarball {
 	return data.CRDTarball
 }
 
+func (data *initData) CustomCertificate() operatorv1alpha1.CustomCertificate {
+	return data.CustomCertificateConfig
+}
+
 func (data *initData) KarmadaVersion() string {
 	return data.karmadaVersion.String()
 }
@@ -278,6 +285,9 @@ func NewInitOptWithKarmada(karmada *operatorv1alpha1.Karmada) InitOpt {
 		if karmada.Spec.CRDTarball != nil {
 			o.CRDTarball = *karmada.Spec.CRDTarball
 		}
+		if karmada.Spec.CustomCertificate != nil {
+			o.CustomCertificateConfig = *karmada.Spec.CustomCertificate
+		}
 	}
 }
 

operator/pkg/tasks/init/cert.go

@@ -22,9 +22,12 @@ import (
 	"fmt"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 	"github.com/karmada-io/karmada/operator/pkg/certs"
+	"github.com/karmada-io/karmada/operator/pkg/constants"
 	"github.com/karmada-io/karmada/operator/pkg/util"
 	"github.com/karmada-io/karmada/operator/pkg/workflow"
 )
@@ -101,6 +104,26 @@ func runCATask(kc *certs.CertConfig) func(d workflow.RunData) error {
 		if kc.CAName != "" {
 			return fmt.Errorf("this function should only be used for CAs, but cert %s has CA %s", kc.Name, kc.CAName)
 		}
+
+		customCertConfig := data.CustomCertificate()
+		if kc.Name == constants.CaCertAndKeyName && customCertConfig.APIServerCACert != nil {
+			secretRef := customCertConfig.APIServerCACert
+			klog.V(4).InfoS("[certs] Loading custom CA certificate", "secret", secretRef.Name, "namespace", secretRef.Namespace)
+
+			certData, keyData, err := loadCACertFromSecret(data.RemoteClient(), secretRef)
+			if err != nil {
+				return fmt.Errorf("failed to load custom CA certificate: %w", err)
+			}
+
+			klog.V(2).InfoS("[certs] Successfully loaded custom CA certificate", "secret", secretRef.Name)
+
+			customKarmadaCert := certs.NewKarmadaCert(kc.Name, kc.CAName, certData, keyData)
+
+			data.AddCert(customKarmadaCert)
+			klog.V(2).InfoS("[certs] Successfully added custom CA certificate to cert store", "certName", kc.Name)
+			return nil
+		}
+
 		klog.V(4).InfoS("[certs] Creating a new certificate authority", "certName", kc.Name)
 
 		cert, err := certs.NewCertificateAuthority(kc)
@@ -115,6 +138,22 @@ func runCATask(kc *certs.CertConfig) func(d workflow.RunData) error {
 	}
 }
 
+func loadCACertFromSecret(client clientset.Interface, ref *operatorv1alpha1.LocalSecretReference) ([]byte, []byte, error) {
+	secret, err := client.CoreV1().Secrets(ref.Namespace).Get(context.TODO(), ref.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to retrieve secret %s/%s: %w", ref.Namespace, ref.Name, err)
+	}
+
+	certData := secret.Data[constants.TLSCertDataKey]
+	keyData := secret.Data[constants.TLSPrivateKeyDataKey]
+
+	if len(certData) == 0 || len(keyData) == 0 {
+		return nil, nil, fmt.Errorf("secret %s/%s is missing required keys: %s and %s", ref.Namespace, ref.Name, constants.TLSCertDataKey, constants.TLSPrivateKeyDataKey)
+	}
+
+	return certData, keyData, nil
+}
+
 func runCertTask(cc, caCert *certs.CertConfig) func(d workflow.RunData) error {
 	return func(r workflow.RunData) error {
 		data, ok := r.(InitData)

operator/pkg/tasks/init/data.go

@@ -36,6 +36,7 @@ type InitData interface {
 	KarmadaClient() clientset.Interface
 	DataDir() string
 	CrdTarball() operatorv1alpha1.CRDTarball
+	CustomCertificate() operatorv1alpha1.CustomCertificate
 	KarmadaVersion() string
 	Components() *operatorv1alpha1.KarmadaComponents
 	FeatureGates() map[string]bool

operator/pkg/tasks/init/test_helpers.go

@@ -46,18 +46,19 @@ func (m *MyTestData) Get() string {
 
 // TestInitData contains the configuration and state required to initialize Karmada components.
 type TestInitData struct {
-	Name                   string
+	Name                    string
-	Namespace              string
+	Namespace               string
-	ControlplaneConfigREST *rest.Config
+	ControlplaneConfigREST  *rest.Config
-	DataDirectory          string
+	DataDirectory           string
-	CrdTarballArchive      operatorv1alpha1.CRDTarball
+	CrdTarballArchive       operatorv1alpha1.CRDTarball
-	KarmadaVersionRelease  string
+	CustomCertificateConfig operatorv1alpha1.CustomCertificate
-	ComponentsUnits        *operatorv1alpha1.KarmadaComponents
+	KarmadaVersionRelease   string
-	FeatureGatesOptions    map[string]bool
+	ComponentsUnits         *operatorv1alpha1.KarmadaComponents
-	RemoteClientConnector  clientset.Interface
+	FeatureGatesOptions     map[string]bool
-	KarmadaClientConnector clientset.Interface
+	RemoteClientConnector   clientset.Interface
-	ControlplaneAddr       string
+	KarmadaClientConnector  clientset.Interface
-	Certs                  []*certs.KarmadaCert
+	ControlplaneAddr        string
+	Certs                   []*certs.KarmadaCert
 }
 
 // Ensure TestInitData implements InitData interface at compile time.
@@ -108,6 +109,11 @@ func (t *TestInitData) CrdTarball() operatorv1alpha1.CRDTarball {
 	return t.CrdTarballArchive
 }
 
+// CustomCertificate returns the custom certificate config.
+func (t *TestInitData) CustomCertificate() operatorv1alpha1.CustomCertificate {
+	return t.CustomCertificateConfig
+}
+
 // KarmadaVersion returns the version of Karmada being used.
 func (t *TestInitData) KarmadaVersion() string {
 	return t.KarmadaVersionRelease