karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for custom API Server CA certificate 

Signed-off-by: Joe Nathan Abellard <contact@jabellard.com>
This commit is contained in:
Joe Nathan Abellard 2024-11-19 17:16:09 -05:00
parent 6795dba9d1
commit 91322e2f5f
12 changed files with 255 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml
View File

@ -3673,6 +3673,32 @@ spec:
type: string
type: object
type: object
customCertificate:
description: |-
CustomCertificate specifies the configuration to customize the certificates
for Karmada components or control the certificate generation process, such as
the algorithm, validity period, etc.
Currently, it only supports customizing the CA certificate for limited components.
properties:
apiServerCACert:
description: |-
APIServerCACert references a Kubernetes secret containing the CA certificate
for component karmada-apiserver.
The secret must contain the following data keys:
- tls.crt: The TLS certificate.
- tls.key: The TLS private key.
If specified, this CA will be used to issue client certificates for
all components that access the APIServer as clients.
properties:
name:
description: Name is the name of resource being referenced.
type: string
namespace:
description: Namespace is the namespace for the resource being
referenced.
type: string
type: object
type: object
featureGates:
additionalProperties:
type: boolean

26
operator/config/crds/operator.karmada.io_karmadas.yaml
View File

@ -3673,6 +3673,32 @@ spec:
type: string
type: object
type: object
customCertificate:
description: |-
CustomCertificate specifies the configuration to customize the certificates
for Karmada components or control the certificate generation process, such as
the algorithm, validity period, etc.
Currently, it only supports customizing the CA certificate for limited components.
properties:
apiServerCACert:
description: |-
APIServerCACert references a Kubernetes secret containing the CA certificate
for component karmada-apiserver.
The secret must contain the following data keys:
- tls.crt: The TLS certificate.
- tls.key: The TLS private key.
If specified, this CA will be used to issue client certificates for
all components that access the APIServer as clients.
properties:
name:
description: Name is the name of resource being referenced.
type: string
namespace:
description: Namespace is the namespace for the resource being
referenced.
type: string
type: object
type: object
featureGates:
additionalProperties:
type: boolean

20
operator/pkg/apis/operator/v1alpha1/type.go
View File

@ -113,6 +113,26 @@ type KarmadaSpec struct {
// By default, the operator will only attempt to download the tarball if it's not yet present in the local cache.
// +optional
CRDTarball *CRDTarball `json:"crdTarball,omitempty"`
// CustomCertificate specifies the configuration to customize the certificates
// for Karmada components or control the certificate generation process, such as
// the algorithm, validity period, etc.
// Currently, it only supports customizing the CA certificate for limited components.
// +optional
CustomCertificate *CustomCertificate `json:"customCertificate,omitempty"`
}
// CustomCertificate holds the configuration for generating the certificate.
type CustomCertificate struct {
// APIServerCACert references a Kubernetes secret containing the CA certificate
// for component karmada-apiserver.
// The secret must contain the following data keys:
// - tls.crt: The TLS certificate.
// - tls.key: The TLS private key.
// If specified, this CA will be used to issue client certificates for
// all components that access the APIServer as clients.
// +optional
APIServerCACert *LocalSecretReference `json:"apiServerCACert,omitempty"`
}
// ImageRegistry represents an image registry as well as the

26
operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
View File

@ -106,6 +106,27 @@ func (in *CommonSettings) DeepCopy() *CommonSettings {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CustomCertificate) DeepCopyInto(out *CustomCertificate) {
*out = *in
if in.APIServerCACert != nil {
in, out := &in.APIServerCACert, &out.APIServerCACert
*out = new(LocalSecretReference)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomCertificate.
func (in *CustomCertificate) DeepCopy() *CustomCertificate {
if in == nil {
return nil
}
out := new(CustomCertificate)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Etcd) DeepCopyInto(out *Etcd) {
*out = *in
@ -637,6 +658,11 @@ func (in *KarmadaSpec) DeepCopyInto(out *KarmadaSpec) {
*out = new(CRDTarball)
(*in).DeepCopyInto(*out)
}
if in.CustomCertificate != nil {
in, out := &in.CustomCertificate, &out.CustomCertificate
*out = new(CustomCertificate)
(*in).DeepCopyInto(*out)
}
return
}

32
operator/pkg/certs/certs.go
View File

@ -212,25 +212,25 @@ func KarmadaCertEtcdClient() *CertConfig {
// KarmadaCert is karmada certificate, it includes certificate basic message.
// we can directly get the byte array of certificate key and cert from the object.
type KarmadaCert struct {
pairName string
caName string
cert []byte
key []byte
PairName string
CAName string
Cert []byte
Key []byte
}
// CertData returns certificate cert data.
func (cert *KarmadaCert) CertData() []byte {
return cert.cert
return cert.Cert
}
// KeyData returns certificate key data.
func (cert *KarmadaCert) KeyData() []byte {
return cert.key
return cert.Key
}
// CertName returns cert file name. its default suffix is ".crt".
func (cert *KarmadaCert) CertName() string {
pair := cert.pairName
pair := cert.PairName
if len(pair) == 0 {
pair = "cert"
}
@ -239,7 +239,7 @@ func (cert *KarmadaCert) CertName() string {
// KeyName returns cert key file name. its default suffix is ".key".
func (cert *KarmadaCert) KeyName() string {
pair := cert.pairName
pair := cert.PairName
if len(pair) == 0 {
pair = "cert"
}
@ -282,10 +282,10 @@ func NewCertificateAuthority(cc *CertConfig) (*KarmadaCert, error) {
}
return &KarmadaCert{
pairName: cc.Name,
caName: cc.CAName,
cert: EncodeCertPEM(cert),
key: encoded,
PairName: cc.Name,
CAName: cc.CAName,
Cert: EncodeCertPEM(cert),
Key: encoded,
}, nil
}
@ -329,10 +329,10 @@ func CreateCertAndKeyFilesWithCA(cc *CertConfig, caCertData, caKeyData []byte) (
}
return &KarmadaCert{
pairName: cc.Name,
caName: cc.CAName,
cert: EncodeCertPEM(cert),
key: encoded,
PairName: cc.Name,
CAName: cc.CAName,
Cert: EncodeCertPEM(cert),
Key: encoded,
}, nil
}

28
operator/pkg/certs/certs_test.go
View File

@ -428,23 +428,23 @@ func TestNewCertificateAuthority(t *testing.T) {
t.Fatal("NewCertificateAuthority() returned nil cert")
}
if cert.pairName != cc.Name {
t.Errorf("expected pairName to be %s, got %s", cc.Name, cert.pairName)
if cert.PairName != cc.Name {
t.Errorf("expected pairName to be %s, got %s", cc.Name, cert.PairName)
}
if cert.caName != cc.CAName {
t.Errorf("expected caName to be %s, got %s", cc.CAName, cert.caName)
if cert.CAName != cc.CAName {
t.Errorf("expected caName to be %s, got %s", cc.CAName, cert.CAName)
}
if cert.cert == nil {
if cert.Cert == nil {
t.Error("expected cert to be non-nil")
}
if cert.key == nil {
if cert.Key == nil {
t.Error("expected key to be non-nil")
}
block, _ := pem.Decode(cert.cert)
block, _ := pem.Decode(cert.Cert)
if block == nil || block.Type != CertificateBlockType {
t.Errorf("expected PEM block type to be %s, got %v", CertificateBlockType, block)
}
@ -524,19 +524,19 @@ func TestCreateCertAndKeyFilesWithCA(t *testing.T) {
t.Fatal("CreateCertAndKeyFilesWithCA() returned nil cert")
}
if cert.cert == nil || cert.key == nil {
if cert.Cert == nil || cert.Key == nil {
t.Error("Expected cert and key to be non-nil")
}
if cert.pairName != certConfig.Name {
t.Errorf("expected pairName to be %s, got %s", certConfig.Name, cert.pairName)
if cert.PairName != certConfig.Name {
t.Errorf("expected pairName to be %s, got %s", certConfig.Name, cert.PairName)
}
if cert.caName != certConfig.CAName {
t.Errorf("expected caName to be %s, got %s", certConfig.CAName, cert.caName)
if cert.CAName != certConfig.CAName {
t.Errorf("expected caName to be %s, got %s", certConfig.CAName, cert.CAName)
}
block, _ := pem.Decode(cert.cert)
block, _ := pem.Decode(cert.Cert)
if block == nil || block.Type != CertificateBlockType {
t.Errorf("expected PEM block type to be %s, got %v", CertificateBlockType, block)
}
@ -566,7 +566,7 @@ func TestNewSignedCert_Success(t *testing.T) {
}
caCert := caCerts[0]
caKey, err := ParsePrivateKeyPEM(caKarmadaCert.key)
caKey, err := ParsePrivateKeyPEM(caKarmadaCert.Key)
if err != nil {
t.Error(err)
}

14
operator/pkg/certs/store.go
View File

@ -66,15 +66,15 @@ func NewCertStore() CertStore {
}
}
// AddCert adds a cert to cert store, the cache key is cert pairName by default.
// AddCert adds a cert to cert store, the cache key is cert PairName by default.
func (store *KarmadaCertStore) AddCert(cert *KarmadaCert) {
store.certs[cert.pairName] = cert
store.certs[cert.PairName] = cert
}
// GetCert get cert from store by cert pairName.
// GetCert get cert from store by cert PairName.
func (store *KarmadaCertStore) GetCert(name string) *KarmadaCert {
for _, c := range store.certs {
if c.pairName == name {
if c.PairName == name {
return c
}
}
@ -105,15 +105,15 @@ func (store *KarmadaCertStore) LoadCertFromSecret(secret *corev1.Secret) error {
kc := store.GetCert(pairName)
if kc == nil {
kc = &KarmadaCert{
pairName: pairName,
PairName: pairName,
}
}
if strings.Contains(name, certExtension) {
kc.cert = data
kc.Cert = data
}
if strings.Contains(name, keyExtension) {
kc.key = data
kc.Key = data
}
store.AddCert(kc)

32
operator/pkg/certs/store_test.go
View File

@ -22,12 +22,12 @@ import (
corev1 "k8s.io/api/core/v1"
)
// Helper function to create a new KarmadaCert with given pairName.
// Helper function to create a new KarmadaCert with given PairName.
func newKarmadaCert(pairName string, certData, keyData []byte) *KarmadaCert {
return &KarmadaCert{
pairName: pairName,
cert: certData,
key: keyData,
PairName: pairName,
Cert: certData,
Key: keyData,
}
}
@ -51,11 +51,11 @@ func TestAddAndGetCert(t *testing.T) {
if retrievedCert == nil {
t.Fatalf("expected to retrieve cert but got nil")
}
if string(retrievedCert.cert) != "certData" {
t.Errorf("expected certData but got %s", string(retrievedCert.cert))
if string(retrievedCert.Cert) != "certData" {
t.Errorf("expected certData but got %s", string(retrievedCert.Cert))
}
if string(retrievedCert.key) != "keyData" {
t.Errorf("expected keyData but got %s", string(retrievedCert.key))
if string(retrievedCert.Key) != "keyData" {
t.Errorf("expected keyData but got %s", string(retrievedCert.Key))
}
}
@ -98,13 +98,13 @@ func TestLoadCertFromSecret(t *testing.T) {
}
cert1 := store.GetCert("cert1")
if cert1 == nil || string(cert1.cert) != "cert1CertData" || string(cert1.key) != "cert1KeyData" {
t.Errorf("cert1 content is incorrect expected cert %s key %s, got cert %s key %s", "cert1CertData", "cert1KeyData", string(cert1.cert), string(cert1.key))
if cert1 == nil || string(cert1.Cert) != "cert1CertData" || string(cert1.Key) != "cert1KeyData" {
t.Errorf("cert1 content is incorrect expected cert %s key %s, got cert %s key %s", "cert1CertData", "cert1KeyData", string(cert1.Cert), string(cert1.Key))
}
cert2 := store.GetCert("cert2")
if cert2 == nil || string(cert2.cert) != "cert2CertData" || string(cert2.key) != "cert2KeyData" {
t.Errorf("cert2 content is incorrect expected cert %s key %s, got cert %s key %s", "cert2CertData", "cert2KeyData", string(cert2.cert), string(cert2.key))
if cert2 == nil || string(cert2.Cert) != "cert2CertData" || string(cert2.Key) != "cert2KeyData" {
t.Errorf("cert2 content is incorrect expected cert %s key %s, got cert %s key %s", "cert2CertData", "cert2KeyData", string(cert2.Cert), string(cert2.Key))
}
}
@ -144,10 +144,10 @@ func TestLoadCertFromSecret_InvalidFormat(t *testing.T) {
}
karmadaCert := store.GetCert(pairName)
if len(karmadaCert.key) != 0 {
t.Errorf("expected the cert data content to be empty but got %v", karmadaCert.cert)
if len(karmadaCert.Key) != 0 {
t.Errorf("expected the cert data content to be empty but got %v", karmadaCert.Cert)
}
if len(karmadaCert.key) != 0 {
t.Errorf("expected the key data content to be empty but got %v", karmadaCert.key)
if len(karmadaCert.Key) != 0 {
t.Errorf("expected the key data content to be empty but got %v", karmadaCert.Key)
}
}

74
operator/pkg/init.go
View File

@ -42,13 +42,14 @@ var (
// InitOptions defines all the init workflow options.
type InitOptions struct {
Name string
Namespace string
Kubeconfig *rest.Config
KarmadaVersion string
CRDTarball operatorv1alpha1.CRDTarball
KarmadaDataDir string
Karmada *operatorv1alpha1.Karmada
Name string
Namespace string
Kubeconfig *rest.Config
KarmadaVersion string
CRDTarball operatorv1alpha1.CRDTarball
CustomCertificateConfig operatorv1alpha1.CustomCertificate
KarmadaDataDir string
Karmada *operatorv1alpha1.Karmada
}
// Validate is used to validate the initOptions before creating initJob.
@ -75,19 +76,20 @@ var _ tasks.InitData = &initData{}
type initData struct {
sync.Once
certs.CertStore
name string
namespace string
karmadaVersion *utilversion.Version
controlplaneConfig *rest.Config
controlplaneAddress string
remoteClient clientset.Interface
karmadaClient clientset.Interface
dnsDomain string
CRDTarball operatorv1alpha1.CRDTarball
karmadaDataDir string
privateRegistry string
featureGates map[string]bool
components *operatorv1alpha1.KarmadaComponents
name string
namespace string
karmadaVersion *utilversion.Version
controlplaneConfig *rest.Config
controlplaneAddress string
remoteClient clientset.Interface
karmadaClient clientset.Interface
dnsDomain string
CRDTarball operatorv1alpha1.CRDTarball
CustomCertificateConfig operatorv1alpha1.CustomCertificate
karmadaDataDir string
privateRegistry string
featureGates map[string]bool
components *operatorv1alpha1.KarmadaComponents
}
// NewInitJob initializes a job with list of init sub-task. and build
@ -165,18 +167,19 @@ func newRunData(opt *InitOptions) (*initData, error) {
}
return &initData{
name: opt.Name,
namespace: opt.Namespace,
karmadaVersion: version,
controlplaneAddress: address,
remoteClient: remoteClient,
CRDTarball: opt.CRDTarball,
karmadaDataDir: opt.KarmadaDataDir,
privateRegistry: privateRegistry,
components: opt.Karmada.Spec.Components,
featureGates: opt.Karmada.Spec.FeatureGates,
dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
CertStore: certs.NewCertStore(),
name: opt.Name,
namespace: opt.Namespace,
karmadaVersion: version,
controlplaneAddress: address,
remoteClient: remoteClient,
CRDTarball: opt.CRDTarball,
CustomCertificateConfig: opt.CustomCertificateConfig,
karmadaDataDir: opt.KarmadaDataDir,
privateRegistry: privateRegistry,
components: opt.Karmada.Spec.Components,
featureGates: opt.Karmada.Spec.FeatureGates,
dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
CertStore: certs.NewCertStore(),
}, nil
}
@ -226,6 +229,10 @@ func (data *initData) CrdTarball() operatorv1alpha1.CRDTarball {
return data.CRDTarball
}
func (data *initData) CustomCertificate() operatorv1alpha1.CustomCertificate {
return data.CustomCertificateConfig
}
func (data *initData) KarmadaVersion() string {
return data.karmadaVersion.String()
}
@ -278,6 +285,9 @@ func NewInitOptWithKarmada(karmada *operatorv1alpha1.Karmada) InitOpt {
if karmada.Spec.CRDTarball != nil {
o.CRDTarball = *karmada.Spec.CRDTarball
}
if karmada.Spec.CustomCertificate != nil {
o.CustomCertificateConfig = *karmada.Spec.CustomCertificate
}
}
}

44
operator/pkg/tasks/init/cert.go
View File

@ -20,6 +20,9 @@ import (
"context"
"errors"
"fmt"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
clientset "k8s.io/client-go/kubernetes"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
@ -101,6 +104,31 @@ func runCATask(kc *certs.CertConfig) func(d workflow.RunData) error {
if kc.CAName != "" {
return fmt.Errorf("this function should only be used for CAs, but cert %s has CA %s", kc.Name, kc.CAName)
}
customCertConfig := data.CustomCertificate()
if kc.Name == constants.CaCertAndKeyName && customCertConfig.APIServerCACert != nil {
secretRef := customCertConfig.APIServerCACert
klog.V(4).InfoS("[certs] Loading custom CA certificate", "secret", secretRef.Name, "namespace", secretRef.Namespace)
certData, keyData, err := loadCACertFromSecret(data.RemoteClient(), secretRef)
if err != nil {
return fmt.Errorf("failed to load custom CA certificate: %w", err)
}
klog.V(2).InfoS("[certs] Successfully loaded custom CA certificate", "secret", secretRef.Name)
customKarmadaCert := &certs.KarmadaCert{
PairName: kc.Name,
CAName: kc.CAName,
Cert: certData,
Key: keyData,
}
data.AddCert(customKarmadaCert)
klog.V(2).InfoS("[certs] Successfully added custom CA certificate to cert store", "certName", kc.Name)
return nil
}
klog.V(4).InfoS("[certs] Creating a new certificate authority", "certName", kc.Name)
cert, err := certs.NewCertificateAuthority(kc)
@ -115,6 +143,22 @@ func runCATask(kc *certs.CertConfig) func(d workflow.RunData) error {
}
}
func loadCACertFromSecret(client clientset.Interface, ref *operatorv1alpha1.LocalSecretReference) ([]byte, []byte, error) {
secret, err := client.CoreV1().Secrets(ref.Namespace).Get(context.TODO(), ref.Name, metav1.GetOptions{})
if err != nil {
return nil, nil, fmt.Errorf("failed to retrieve secret %s/%s: %w", ref.Namespace, ref.Name, err)
}
certData := secret.Data[constants.TLSCertDataKey]
keyData := secret.Data[constants.TLSPrivateKeyDataKey]
if len(certData) == 0 || len(keyData) == 0 {
return nil, nil, fmt.Errorf("secret %s/%s is missing required keys: %s and %s", ref.Namespace, ref.Name, constants.TLSCertDataKey, constants.TLSPrivateKeyDataKey)
}
return certData, keyData, nil
}
func runCertTask(cc, caCert *certs.CertConfig) func(d workflow.RunData) error {
return func(r workflow.RunData) error {
data, ok := r.(InitData)

1
operator/pkg/tasks/init/data.go
View File

@ -36,6 +36,7 @@ type InitData interface {
KarmadaClient() clientset.Interface
DataDir() string
CrdTarball() operatorv1alpha1.CRDTarball
CustomCertificate() operatorv1alpha1.CustomCertificate
KarmadaVersion() string
Components() *operatorv1alpha1.KarmadaComponents
FeatureGates() map[string]bool

29
operator/pkg/tasks/init/test_helpers.go
View File

@ -46,18 +46,19 @@ func (m *MyTestData) Get() string {
// TestInitData contains the configuration and state required to initialize Karmada components.
type TestInitData struct {
Name string
Namespace string
ControlplaneConfigREST *rest.Config
DataDirectory string
CrdTarballArchive operatorv1alpha1.CRDTarball
KarmadaVersionRelease string
ComponentsUnits *operatorv1alpha1.KarmadaComponents
FeatureGatesOptions map[string]bool
RemoteClientConnector clientset.Interface
KarmadaClientConnector clientset.Interface
ControlplaneAddr string
Certs []*certs.KarmadaCert
Name string
Namespace string
ControlplaneConfigREST *rest.Config
DataDirectory string
CrdTarballArchive operatorv1alpha1.CRDTarball
CustomCertificateConfig operatorv1alpha1.CustomCertificate
KarmadaVersionRelease string
ComponentsUnits *operatorv1alpha1.KarmadaComponents
FeatureGatesOptions map[string]bool
RemoteClientConnector clientset.Interface
KarmadaClientConnector clientset.Interface
ControlplaneAddr string
Certs []*certs.KarmadaCert
}
// Ensure TestInitData implements InitData interface at compile time.
@ -108,6 +109,10 @@ func (t *TestInitData) CrdTarball() operatorv1alpha1.CRDTarball {
return t.CrdTarballArchive
}
func (t *TestInitData) CustomCertificate() operatorv1alpha1.CustomCertificate {
return t.CustomCertificateConfig
}
// KarmadaVersion returns the version of Karmada being used.
func (t *TestInitData) KarmadaVersion() string {
return t.KarmadaVersionRelease