feat: manager init cert for grpc server (#1603)

Signed-off-by: Gaius <gaius.qi@gmail.com>

client/daemon/daemon.go

@@ -141,7 +141,7 @@ func New(opt *config.DaemonOption, d dfpath.Dfpath) (Daemon, error) {
 				Logger:       zapadapter.New(logger.CoreLogger.Desugar()),
 				Cache: cache.NewCertifyMutliCache(
 					certify.NewMemCache(),
-					certify.DirCache(path.Join(d.CacheDir(), "certs"))),
+					certify.DirCache(path.Join(d.CacheDir(), cache.CertifyCacheDirName))),
 			}
 
 			// issue a certificate to reduce first time delay

manager/config/config.go

@@ -214,10 +214,10 @@ type MetricsConfig struct {
 }
 
 type TCPListenConfig struct {
-	// Listen stands listen interface, like: 0.0.0.0, 192.168.0.1.
+	// Listen is listen interface, like: 0.0.0.0, 192.168.0.1.
 	Listen string `mapstructure:"listen" yaml:"listen"`
 
-	// PortRange stands listen port.
+	// PortRange is listen port.
 	PortRange TCPListenPortRange `yaml:"port" mapstructure:"port"`
 }
 

manager/manager.go

@@ -22,10 +22,13 @@ import (
 	"embed"
 	"io/fs"
 	"net/http"
+	"path"
 	"time"
 
 	"github.com/gin-contrib/static"
+	"github.com/johanbrandhorst/certify"
 	"google.golang.org/grpc"
+	zapadapter "logur.dev/adapter/zap"
 
 	logger "d7y.io/dragonfly/v2/internal/dflog"
 	"d7y.io/dragonfly/v2/manager/cache"
@@ -38,7 +41,10 @@ import (
 	"d7y.io/dragonfly/v2/manager/rpcserver"
 	"d7y.io/dragonfly/v2/manager/searcher"
 	"d7y.io/dragonfly/v2/manager/service"
+	pkgcache "d7y.io/dragonfly/v2/pkg/cache"
 	"d7y.io/dragonfly/v2/pkg/dfpath"
+	"d7y.io/dragonfly/v2/pkg/issuer"
+	"d7y.io/dragonfly/v2/pkg/net/ip"
 	"d7y.io/dragonfly/v2/pkg/objectstorage"
 	"d7y.io/dragonfly/v2/pkg/rpc"
 )
@@ -154,7 +160,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 		return nil, err
 	}
 
-	// Initialize global certificate.
+	// Initialize signing certificate and tls credentials of grpc server.
 	var options []rpcserver.Option
 	if cfg.Security.Enable {
 		cert, err := tls.X509KeyPair([]byte(cfg.Security.CACert), []byte(cfg.Security.CAKey))
@@ -162,7 +168,30 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 			return nil, err
 		}
 
-		options = append(options, rpcserver.WithCertificate(&cert))
+		// Manager GRPC server's tls varify must be false. If ClientCAs are required for client verification,
+		// the client cannot call the IssueCertificate api.
+		transportCredentials, err := rpc.NewServerCredentialsByCertify(false, &cert, &certify.Certify{
+			CommonName:   ip.IPv4,
+			Issuer:       issuer.NewDragonflyManagerIssuer(&cert),
+			RenewBefore:  time.Hour,
+			CertConfig:   nil,
+			IssueTimeout: 0,
+			Logger:       zapadapter.New(logger.CoreLogger.Desugar()),
+			Cache: pkgcache.NewCertifyMutliCache(
+				certify.NewMemCache(),
+				certify.DirCache(path.Join(d.CacheDir(), pkgcache.CertifyCacheDirName))),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		options = append(
+			options,
+			// Set ca certificate for issuing certificate.
+			rpcserver.WithSelfSignedCert(&cert),
+			// Set tls credentials for grpc server.
+			rpcserver.WithGRPCServerOptions([]grpc.ServerOption{grpc.Creds(transportCredentials)}),
+		)
 	}
 
 	// Initialize GRPC server

manager/rpcserver/cert.go

@@ -36,8 +36,11 @@ import (
 	logger "d7y.io/dragonfly/v2/internal/dflog"
 )
 
+// defaultValidityDuration is default validity duration of certificate.
+const defaultValidityDuration = 365 * 24 * time.Hour
+
 func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.CertificateRequest) (*securityv1.CertificateResponse, error) {
-	if s.cert == nil {
+	if s.selfSignedCert == nil {
 		return nil, status.Errorf(codes.Unavailable, "ca is missing for this manager instance")
 	}
 
@@ -86,7 +89,7 @@ func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.Certifica
 	now := time.Now()
 	duration := time.Duration(req.ValidityDuration) * time.Second
 	if duration == 0 {
-		duration = time.Hour
+		duration = defaultValidityDuration
 	}
 
 	logger.Infof("valid csr: %#v", csr.Subject)
@@ -102,18 +105,18 @@ func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.Certifica
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
 	}
 
-	cert, err := x509.CreateCertificate(rand.Reader, &template, s.x509Cert, csr.PublicKey, s.cert.PrivateKey)
+	cert, err := x509.CreateCertificate(rand.Reader, &template, s.selfSignedCert.X509Cert, csr.PublicKey, s.selfSignedCert.TLSCert.PrivateKey)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate certificate, error: %s", err)
 	}
 
-	// Encode into PEM format.
+	// Build the certificate chain.
 	var certPEM bytes.Buffer
 	if err = pem.Encode(&certPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
 		return nil, err
 	}
 
 	return &securityv1.CertificateResponse{
-		CertificateChain: append([]string{certPEM.String()}, s.certChain...),
+		CertificateChain: append([]string{certPEM.String()}, s.selfSignedCert.PEMCertChain...),
 	}, nil
 }

manager/rpcserver/cert_test.go

@@ -89,7 +89,7 @@ func TestIssueCertificate(t *testing.T) {
 					DB:  &gorm.DB{},
 					RDB: &redis.Client{},
 				},
-				nil, nil, nil, nil, WithCertificate(&ca))
+				nil, nil, nil, nil, WithSelfSignedCert(&ca))
 			require.Nilf(err, "newServer should be ok")
 
 			ctx := peer.NewContext(
@@ -110,7 +110,7 @@ func TestIssueCertificate(t *testing.T) {
 
 			assert.Nilf(err, "IssueCertificate should be ok")
 			assert.NotNilf(resp, "IssueCertificate should not be nil")
-			assert.Equal(len(resp.CertificateChain), len(server.certChain)+1)
+			assert.Equal(len(resp.CertificateChain), 2)
 
 			cert := readCert(resp.CertificateChain[0])
 			assert.Equal(len(cert.IPAddresses), 1)

manager/rpcserver/rpcserver.go

@@ -50,6 +50,18 @@ import (
 	managerserver "d7y.io/dragonfly/v2/pkg/rpc/manager/server"
 )
 
+// SelfSignedCert is self signed certificate.
+type SelfSignedCert struct {
+	// TLSCert is certificate of tls.
+	TLSCert *tls.Certificate
+
+	// X509Cert is certificate of x509.
+	X509Cert *x509.Certificate
+
+	// PEMCertChain is certificate chain of pem.
+	PEMCertChain []string
+}
+
 // Server is grpc server.
 type Server struct {
 	// Manager configuration.
@@ -76,39 +88,35 @@ type Server struct {
 	// serverOptions is server options of grpc.
 	serverOptions []grpc.ServerOption
 
-	// cert certificates to sign certificates.
+	// selfSignedCert is self signed certificate.
-	cert *tls.Certificate
+	selfSignedCert *SelfSignedCert
-
-	// x509Cert certificates to sign certificates.
-	x509Cert *x509.Certificate
-
-	// certChain is PEM-encoded certificate chain.
-	certChain []string
 }
 
 // Option is a functional option for rpc server.
 type Option func(s *Server) error
 
-// WithCertificate set the root tls certificate, x509 certificate and PEM-encoded certificate chain.
+// WithCertificate set the self signed certificate for server.
-func WithCertificate(cert *tls.Certificate) Option {
+func WithSelfSignedCert(tlsCert *tls.Certificate) Option {
 	return func(s *Server) error {
-		s.cert = cert
+		x509CACert, err := x509.ParseCertificate(tlsCert.Certificate[0])
-
-		// Parse x509 certificate from tls certificate.
-		var err error
-		s.x509Cert, err = x509.ParseCertificate(cert.Certificate[0])
 		if err != nil {
 			return err
 		}
 
-		// Initialize PEM-encoded certificate chain from tls certificate.
+		var pemCertChain []string
-		for _, cert := range cert.Certificate {
+		for _, cert := range tlsCert.Certificate {
-			var certChainPEM bytes.Buffer
+			var certPEM bytes.Buffer
-			if err = pem.Encode(&certChainPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
+			if err := pem.Encode(&certPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
 				return err
 			}
 
-			s.certChain = append(s.certChain, certChainPEM.String())
+			pemCertChain = append(pemCertChain, certPEM.String())
+		}
+
+		s.selfSignedCert = &SelfSignedCert{
+			TLSCert:      tlsCert,
+			X509Cert:     x509CACert,
+			PEMCertChain: pemCertChain,
 		}
 
 		return nil

pkg/cache/certify.go

@@ -23,18 +23,23 @@ import (
 	"github.com/johanbrandhorst/certify"
 )
 
+const (
+	// CertifyCacheDirName is dir name of certify cache.
+	CertifyCacheDirName = "certs"
+)
+
 type certifyCache struct {
 	caches []certify.Cache
 }
 
 // NewCertifyMutliCache returns a certify.Cache with multiple caches
 // Such as, cache.NewCertifyMutliCache(certify.NewMemCache(), certify.DirCache("certs"))
-// This multiple cache will get certs from mem cache first, then dir cache to avoid read from filesystem every times
+// This multiple cache will get certs from mem cache first, then dir cache to avoid read from filesystem every times.
 func NewCertifyMutliCache(caches ...certify.Cache) certify.Cache {
 	return &certifyCache{caches: caches}
 }
 
-// Get gets cert from cache one by one, if found, puts it back to all previous cachs
+// Get gets cert from cache one by one, if found, puts it back to all previous caches.
 func (c *certifyCache) Get(ctx context.Context, key string) (cert *tls.Certificate, err error) {
 	var foundCacheIdx int = -1
 	for i, cache := range c.caches {
@@ -55,7 +60,7 @@ func (c *certifyCache) Get(ctx context.Context, key string) (cert *tls.Certifica
 	return nil, certify.ErrCacheMiss
 }
 
-// Put puts cert to all caches
+// Put puts cert to all caches.
 func (c *certifyCache) Put(ctx context.Context, key string, cert *tls.Certificate) error {
 	for _, cache := range c.caches {
 		if err := cache.Put(ctx, key, cert); err != nil {
@@ -65,7 +70,7 @@ func (c *certifyCache) Put(ctx context.Context, key string, cert *tls.Certificat
 	return nil
 }
 
-// Delete deletes cert from all caches
+// Delete deletes cert from all caches.
 func (c *certifyCache) Delete(ctx context.Context, key string) error {
 	for _, cache := range c.caches {
 		if err := cache.Delete(ctx, key); err != nil {

pkg/issuer/dragonfly_manager.go

@@ -0,0 +1,165 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package issuer
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"net/url"
+	"time"
+
+	"github.com/johanbrandhorst/certify"
+
+	"d7y.io/dragonfly/v2/pkg/net/ip"
+)
+
+var (
+	// defaultSubjectCommonName is default common name of subject.
+	defaultSubjectCommonName = "manager"
+
+	// defaultSubjectOrganization is default organization of subject.
+	defaultSubjectOrganization = []string{"Dragonfly"}
+
+	// defaultSubjectOrganizationalUnit is default organizational unit of subject.
+	defaultSubjectOrganizationalUnit = []string{"Manager"}
+
+	// defaultIPAddresses is default ip addresses of certificate.
+	defaultIPAddresses = []net.IP{net.ParseIP(ip.IPv4)}
+
+	// defaultDNSNames is default dns names of certificate.
+	defaultDNSNames = []string{"dragonfly-manager", "dragonfly-manager.dragonfly-system.svc"}
+
+	// defaultValidityDuration is default validity duration of certificate.
+	defaultValidityDuration = 10 * 365 * 24 * time.Hour
+)
+
+// GC provides issuer function.
+type dragonflyManagerIssuer struct {
+	tlsCACert        *tls.Certificate
+	dnsNames         []string
+	emailAddresses   []string
+	ipAddresses      []net.IP
+	uris             []*url.URL
+	validityDuration time.Duration
+}
+
+// Option is a functional option for configuring the dragonflyManagerIssuer.
+type Option func(i *dragonflyManagerIssuer)
+
+// WithDNSNames set the dnsNames for dragonflyManagerIssuer.
+func WithDNSNames(dnsNames []string) Option {
+	return func(i *dragonflyManagerIssuer) {
+		i.dnsNames = dnsNames
+	}
+}
+
+// WithEmailAddresses set the emailAddresses for dragonflyManagerIssuer.
+func WithEmailAddresses(emailAddrs []string) Option {
+	return func(i *dragonflyManagerIssuer) {
+		i.emailAddresses = emailAddrs
+	}
+}
+
+// WithIPAddresses set the ipAddresses for dragonflyManagerIssuer.
+func WithIPAddresses(ipAddrs []net.IP) Option {
+	return func(i *dragonflyManagerIssuer) {
+		i.ipAddresses = ipAddrs
+	}
+}
+
+// WithURIs set the uris for dragonflyManagerIssuer.
+func WithURIs(uris []*url.URL) Option {
+	return func(i *dragonflyManagerIssuer) {
+		i.uris = uris
+	}
+}
+
+// WithValidityDuration set the validityDuration for dragonflyManagerIssuer.
+func WithValidityDuration(d time.Duration) Option {
+	return func(i *dragonflyManagerIssuer) {
+		i.validityDuration = d
+	}
+}
+
+// NewDragonflyManagerIssuer returns a new certify.Issuer instence.
+func NewDragonflyManagerIssuer(tlsCACert *tls.Certificate, opts ...Option) certify.Issuer {
+	i := &dragonflyManagerIssuer{
+		tlsCACert:        tlsCACert,
+		dnsNames:         defaultDNSNames,
+		ipAddresses:      defaultIPAddresses,
+		validityDuration: defaultValidityDuration,
+	}
+
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	return i
+}
+
+// Issue returns tls Certificate of issuing.
+func (i *dragonflyManagerIssuer) Issue(ctx context.Context, commonName string, certConfig *certify.CertConfig) (*tls.Certificate, error) {
+	x509CACert, err := x509.ParseCertificate(i.tlsCACert.Certificate[0])
+	if err != nil {
+		return nil, err
+	}
+
+	pk, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, err
+	}
+
+	serial, err := rand.Int(rand.Reader, (&big.Int{}).Exp(big.NewInt(2), big.NewInt(159), nil))
+	if err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	template := x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName:         defaultSubjectCommonName,
+			Organization:       defaultSubjectOrganization,
+			OrganizationalUnit: defaultSubjectOrganizationalUnit,
+		},
+		DNSNames:              i.dnsNames,
+		EmailAddresses:        i.emailAddresses,
+		IPAddresses:           i.ipAddresses,
+		URIs:                  i.uris,
+		NotBefore:             now.Add(-10 * time.Minute).UTC(),
+		NotAfter:              now.Add(i.validityDuration).UTC(),
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageDataEncipherment | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	cert, err := x509.CreateCertificate(rand.Reader, &template, x509CACert, &pk.PublicKey, i.tlsCACert.PrivateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tls.Certificate{
+		Certificate: append([][]byte{cert}, i.tlsCACert.Certificate...),
+		PrivateKey:  pk,
+	}, nil
+}

pkg/rpc/credential.go

@@ -0,0 +1,50 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rpc
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+
+	"github.com/johanbrandhorst/certify"
+	"google.golang.org/grpc/credentials"
+)
+
+// NewServerCredentialsByCertify returns server transport credentials by certify.
+func NewServerCredentialsByCertify(tlsVerify bool, tlsCACert *tls.Certificate, certifyClient *certify.Certify) (credentials.TransportCredentials, error) {
+	if !tlsVerify {
+		return credentials.NewTLS(&tls.Config{
+			GetCertificate: certifyClient.GetCertificate,
+		}), nil
+	}
+
+	certPool := x509.NewCertPool()
+	for _, cert := range tlsCACert.Certificate {
+		if !certPool.AppendCertsFromPEM(cert) {
+			return nil, errors.New("invalid CA Cert")
+		}
+	}
+
+	tlsConfig := &tls.Config{
+		GetCertificate: certifyClient.GetCertificate,
+		ClientAuth:     tls.RequireAndVerifyClientCert,
+		ClientCAs:      certPool,
+	}
+
+	return credentials.NewTLS(tlsConfig), nil
+}