karmada/pkg/controllers/certificate/cert_rotation_controller.go

/*
Copyright 2022 The Karmada Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certificate

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"time"

	certificatesv1 "k8s.io/api/certificates/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8srand "k8s.io/apimachinery/pkg/util/rand"
	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
	"k8s.io/client-go/tools/record"
	certutil "k8s.io/client-go/util/cert"
	"k8s.io/client-go/util/keyutil"
	"k8s.io/klog/v2"
	controllerruntime "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/builder"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/predicate"

	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
	"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
	"github.com/karmada-io/karmada/pkg/util"
	"github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager"
)

const (
	// CertRotationControllerName is the controller name that will be used when reporting events and metrics.
	CertRotationControllerName = "cert-rotation-controller"

	// SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client-kubelet' can sign the csr automatically
	SignerName = "kubernetes.io/kube-apiserver-client-kubelet"

	// KarmadaKubeconfigName is the name of the secret containing karmada-agent certificate.
	KarmadaKubeconfigName = "karmada-kubeconfig"
)

// CertRotationController is to rotate certificates.
type CertRotationController struct {
	client.Client        // used to operate cluster resources in the control plane.
	KubeClient           clientset.Interface
	EventRecorder        record.EventRecorder
	RESTMapper           meta.RESTMapper
	ClusterClient        *util.ClusterClient
	ClusterClientSetFunc func(string, client.Client, *util.ClientOption) (*util.ClusterClient, error)
	// ClusterClientOption holds the attributes that should be injected to a Kubernetes client.
	ClusterClientOption *util.ClientOption
	PredicateFunc       predicate.Predicate
	InformerManager     genericmanager.MultiClusterInformerManager
	RatelimiterOptions  ratelimiterflag.Options

	// CertRotationCheckingInterval defines the interval of checking if the certificate need to be rotated.
	CertRotationCheckingInterval time.Duration
	// KarmadaKubeconfigNamespace is the namespace of the secret containing karmada-agent certificate.
	KarmadaKubeconfigNamespace string
	// CertRotationRemainingTimeThreshold defines the threshold of remaining time of the valid certificate.
	// If the ratio of remaining time to total time is less than or equal to this threshold, the certificate rotation starts.
	CertRotationRemainingTimeThreshold float64
}

// Reconcile performs a full reconciliation for the object referred to by the Request.
// The Controller will requeue the Request to be processed again if an error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (c *CertRotationController) Reconcile(ctx context.Context, req controllerruntime.Request) (controllerruntime.Result, error) {
	klog.V(4).Infof("Rotating the certificate of karmada-agent for the member cluster: %s", req.NamespacedName.Name)

	var err error

	cluster := &clusterv1alpha1.Cluster{}
	if err := c.Client.Get(ctx, req.NamespacedName, cluster); err != nil {
		// The resource may no longer exist, in which case we stop processing.
		if apierrors.IsNotFound(err) {
			return controllerruntime.Result{}, nil
		}

		return controllerruntime.Result{}, err
	}

	if !cluster.DeletionTimestamp.IsZero() {
		return controllerruntime.Result{}, nil
	}

	// create a ClusterClient for the given member cluster
	c.ClusterClient, err = c.ClusterClientSetFunc(cluster.Name, c.Client, c.ClusterClientOption)
	if err != nil {
		klog.Errorf("Failed to create a ClusterClient for the given member cluster: %s, err is: %v", cluster.Name, err)
		return controllerruntime.Result{}, err
	}

	secret, err := c.ClusterClient.KubeClient.CoreV1().Secrets(c.KarmadaKubeconfigNamespace).Get(ctx, KarmadaKubeconfigName, metav1.GetOptions{})
	if err != nil {
		klog.Errorf("failed to get karmada kubeconfig secret: %v", err)
		return controllerruntime.Result{}, err
	}

	if err = c.syncCertRotation(ctx, secret); err != nil {
		klog.Errorf("Failed to rotate the certificate of karmada-agent for the given member cluster: %s, err is: %v", cluster.Name, err)
		return controllerruntime.Result{}, err
	}

	return controllerruntime.Result{RequeueAfter: c.CertRotationCheckingInterval}, nil
}

// SetupWithManager creates a controller and register to controller manager.
func (c *CertRotationController) SetupWithManager(mgr controllerruntime.Manager) error {
	return controllerruntime.NewControllerManagedBy(mgr).
		Named(CertRotationControllerName).
		For(&clusterv1alpha1.Cluster{}, builder.WithPredicates(c.PredicateFunc)).
		WithEventFilter(predicate.GenerationChangedPredicate{}).
		WithOptions(controller.Options{
			RateLimiter: ratelimiterflag.DefaultControllerRateLimiter(c.RatelimiterOptions),
		}).
		Complete(c)
}

func (c *CertRotationController) syncCertRotation(ctx context.Context, secret *corev1.Secret) error {
	karmadaKubeconfig, err := getKubeconfigFromSecret(secret)
	if err != nil {
		return err
	}

	clusterName := karmadaKubeconfig.Contexts[karmadaKubeconfig.CurrentContext].AuthInfo
	if clusterName == "" {
		return fmt.Errorf("failed to get cluster name, the current context is %s", karmadaKubeconfig.CurrentContext)
	}

	oldCertData := karmadaKubeconfig.AuthInfos[clusterName].ClientCertificateData

	shouldRotate, err := c.shouldRotateCert(oldCertData)
	if err != nil {
		return err
	}
	if !shouldRotate {
		return nil
	}

	oldCert, err := certutil.ParseCertsPEM(oldCertData)
	if err != nil {
		return fmt.Errorf("unable to parse old certificate: %v", err)
	}

	// create a new private key
	keyData, err := keyutil.MakeEllipticPrivateKeyPEM()
	if err != nil {
		return err
	}

	privateKey, err := keyutil.ParsePrivateKeyPEM(keyData)
	if err != nil {
		return fmt.Errorf("invalid private key for certificate request: %v", err)
	}

	csr, err := c.createCSRInControlPlane(ctx, clusterName, privateKey, oldCert)
	if err != nil {
		return fmt.Errorf("failed to create csr in control plane, err is: %v", err)
	}

	var newCertData []byte
	klog.V(1).Infof("Waiting for the client certificate to be issued")
	err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 5*time.Minute, false, func(context.Context) (done bool, err error) {
		csr, err := c.KubeClient.CertificatesV1().CertificateSigningRequests().Get(ctx, csr, metav1.GetOptions{})
		if err != nil {
			return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", clusterName, err)
		}

		if csr.Status.Certificate != nil {
			klog.V(1).Infof("Signing certificate successfully")
			newCertData = csr.Status.Certificate
			return true, nil
		}

		klog.V(1).Infof("Waiting for the client certificate to be issued")
		return false, nil
	})
	if err != nil {
		return err
	}

	karmadaKubeconfig.AuthInfos[clusterName].ClientCertificateData = newCertData
	karmadaKubeconfig.AuthInfos[clusterName].ClientKeyData = keyData

	karmadaKubeconfigBytes, err := clientcmd.Write(*karmadaKubeconfig)
	if err != nil {
		return fmt.Errorf("failed to serialize karmada-agent kubeConfig. %v", err)
	}

	secret.Data["karmada-kubeconfig"] = karmadaKubeconfigBytes
	// Update the karmada-kubeconfig secret in the member cluster.
	if _, err := c.ClusterClient.KubeClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
		return fmt.Errorf("unable to update secret, err: %w", err)
	}

	newCert, err := certutil.ParseCertsPEM(newCertData)
	if err != nil {
		klog.Errorf("Unable to parse new certificate: %v", err)
		return err
	}

	klog.V(4).Infof("The certificate has been rotated successfully, new expiration time is %v", newCert[0].NotAfter)

	return nil
}

func (c *CertRotationController) createCSRInControlPlane(ctx context.Context, clusterName string, privateKey interface{}, oldCert []*x509.Certificate) (string, error) {
	csrData, err := certutil.MakeCSR(privateKey, &oldCert[0].Subject, nil, nil)
	if err != nil {
		return "", fmt.Errorf("unable to generate certificate request: %v", err)
	}

	csrName := clusterName + "-" + k8srand.String(5)

	// Expiration of the new certificate is same with the old certificate
	certExpirationSeconds := int32((oldCert[0].NotAfter.Sub(oldCert[0].NotBefore)).Seconds())

	certificateSigningRequest := &certificatesv1.CertificateSigningRequest{
		ObjectMeta: metav1.ObjectMeta{
			Name: csrName,
		},
		Spec: certificatesv1.CertificateSigningRequestSpec{
			Request:           csrData,
			SignerName:        SignerName,
			ExpirationSeconds: &certExpirationSeconds,
			Usages: []certificatesv1.KeyUsage{
				certificatesv1.UsageDigitalSignature,
				certificatesv1.UsageKeyEncipherment,
				certificatesv1.UsageClientAuth,
			},
		},
	}

	_, err = c.KubeClient.CertificatesV1().CertificateSigningRequests().Create(ctx, certificateSigningRequest, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("unable to create certificate request in control plane: %v", err)
	}

	return csrName, nil
}

func getKubeconfigFromSecret(secret *corev1.Secret) (*clientcmdapi.Config, error) {
	if secret.Data == nil {
		return nil, fmt.Errorf("no client certificate found in secret %q", secret.Namespace+"/"+secret.Name)
	}

	karmadaKubeconfigData, ok := secret.Data[KarmadaKubeconfigName]
	if !ok {
		return nil, fmt.Errorf("no karmada kubeconfig found in secret %q", secret.Namespace+"/"+secret.Name)
	}

	karmadaKubeconfig, err := clientcmd.Load(karmadaKubeconfigData)
	if err != nil {
		return nil, err
	}

	return karmadaKubeconfig, nil
}

func (c *CertRotationController) shouldRotateCert(certData []byte) (bool, error) {
	notBefore, notAfter, err := getCertValidityPeriod(certData)
	if err != nil {
		return false, err
	}

	total := notAfter.Sub(*notBefore)
	remaining := time.Until(*notAfter)
	klog.V(4).Infof("The certificate of karmada-agent: time total=%v, remaining=%v, remaining/total=%v", total, remaining, remaining.Seconds()/total.Seconds())

	if remaining.Seconds()/total.Seconds() > c.CertRotationRemainingTimeThreshold {
		// Do nothing if the certificate of karmada-agent is valid and has more than a valid threshold of its life remaining
		klog.V(4).Infof("The certificate of karmada-agent is valid and has more than %.2f%% of its life remaining", c.CertRotationRemainingTimeThreshold*100)
		return false, nil
	}

	klog.V(4).Infof("The certificate of karmada-agent has less than or equal %.2f%% of its life remaining and need to be rotated", c.CertRotationRemainingTimeThreshold*100)
	return true, nil
}

// getCertValidityPeriod returns the validity period of the certificate
func getCertValidityPeriod(certData []byte) (*time.Time, *time.Time, error) {
	certs, err := certutil.ParseCertsPEM(certData)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to parse TLS certificates: %v", err)
	}

	if len(certs) == 0 {
		return nil, nil, errors.New("no cert found in certificate")
	}

	// find out the validity period for all certs in the certificate chain
	var notBefore, notAfter *time.Time
	for index, cert := range certs {
		if index == 0 {
			notBefore = &cert.NotBefore
			notAfter = &cert.NotAfter
			continue
		}

		if notBefore.Before(cert.NotBefore) {
			notBefore = &cert.NotBefore
		}

		if notAfter.After(cert.NotAfter) {
			notAfter = &cert.NotAfter
		}
	}

	return notBefore, notAfter, nil
}