From 9ab33635c53e6a80514f2347728edd37cc9b4fd1 Mon Sep 17 00:00:00 2001 From: Gaius Date: Tue, 30 Aug 2022 14:07:12 +0800 Subject: [PATCH] feat: manager init cert for grpc server (#1603) Signed-off-by: Gaius --- client/daemon/daemon.go | 2 +- manager/config/config.go | 4 +- manager/manager.go | 33 ++++++- manager/rpcserver/cert.go | 13 ++- manager/rpcserver/cert_test.go | 4 +- manager/rpcserver/rpcserver.go | 48 ++++++---- pkg/cache/certify.go | 13 ++- pkg/issuer/dragonfly_manager.go | 165 ++++++++++++++++++++++++++++++++ pkg/rpc/credential.go | 50 ++++++++++ 9 files changed, 296 insertions(+), 36 deletions(-) create mode 100644 pkg/issuer/dragonfly_manager.go create mode 100644 pkg/rpc/credential.go diff --git a/client/daemon/daemon.go b/client/daemon/daemon.go index 1c7f7cc76..7976ed601 100644 --- a/client/daemon/daemon.go +++ b/client/daemon/daemon.go @@ -141,7 +141,7 @@ func New(opt *config.DaemonOption, d dfpath.Dfpath) (Daemon, error) { Logger: zapadapter.New(logger.CoreLogger.Desugar()), Cache: cache.NewCertifyMutliCache( certify.NewMemCache(), - certify.DirCache(path.Join(d.CacheDir(), "certs"))), + certify.DirCache(path.Join(d.CacheDir(), cache.CertifyCacheDirName))), } // issue a certificate to reduce first time delay diff --git a/manager/config/config.go b/manager/config/config.go index 4ab2155ac..30065b090 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -214,10 +214,10 @@ type MetricsConfig struct { } type TCPListenConfig struct { - // Listen stands listen interface, like: 0.0.0.0, 192.168.0.1. + // Listen is listen interface, like: 0.0.0.0, 192.168.0.1. Listen string `mapstructure:"listen" yaml:"listen"` - // PortRange stands listen port. + // PortRange is listen port. PortRange TCPListenPortRange `yaml:"port" mapstructure:"port"` } diff --git a/manager/manager.go b/manager/manager.go index 4c1663133..d07ae7fb1 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -22,10 +22,13 @@ import ( "embed" "io/fs" "net/http" + "path" "time" "github.com/gin-contrib/static" + "github.com/johanbrandhorst/certify" "google.golang.org/grpc" + zapadapter "logur.dev/adapter/zap" logger "d7y.io/dragonfly/v2/internal/dflog" "d7y.io/dragonfly/v2/manager/cache" @@ -38,7 +41,10 @@ import ( "d7y.io/dragonfly/v2/manager/rpcserver" "d7y.io/dragonfly/v2/manager/searcher" "d7y.io/dragonfly/v2/manager/service" + pkgcache "d7y.io/dragonfly/v2/pkg/cache" "d7y.io/dragonfly/v2/pkg/dfpath" + "d7y.io/dragonfly/v2/pkg/issuer" + "d7y.io/dragonfly/v2/pkg/net/ip" "d7y.io/dragonfly/v2/pkg/objectstorage" "d7y.io/dragonfly/v2/pkg/rpc" ) @@ -154,7 +160,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) { return nil, err } - // Initialize global certificate. + // Initialize signing certificate and tls credentials of grpc server. var options []rpcserver.Option if cfg.Security.Enable { cert, err := tls.X509KeyPair([]byte(cfg.Security.CACert), []byte(cfg.Security.CAKey)) @@ -162,7 +168,30 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) { return nil, err } - options = append(options, rpcserver.WithCertificate(&cert)) + // Manager GRPC server's tls varify must be false. If ClientCAs are required for client verification, + // the client cannot call the IssueCertificate api. + transportCredentials, err := rpc.NewServerCredentialsByCertify(false, &cert, &certify.Certify{ + CommonName: ip.IPv4, + Issuer: issuer.NewDragonflyManagerIssuer(&cert), + RenewBefore: time.Hour, + CertConfig: nil, + IssueTimeout: 0, + Logger: zapadapter.New(logger.CoreLogger.Desugar()), + Cache: pkgcache.NewCertifyMutliCache( + certify.NewMemCache(), + certify.DirCache(path.Join(d.CacheDir(), pkgcache.CertifyCacheDirName))), + }) + if err != nil { + return nil, err + } + + options = append( + options, + // Set ca certificate for issuing certificate. + rpcserver.WithSelfSignedCert(&cert), + // Set tls credentials for grpc server. + rpcserver.WithGRPCServerOptions([]grpc.ServerOption{grpc.Creds(transportCredentials)}), + ) } // Initialize GRPC server diff --git a/manager/rpcserver/cert.go b/manager/rpcserver/cert.go index 1114d445e..8be893b66 100644 --- a/manager/rpcserver/cert.go +++ b/manager/rpcserver/cert.go @@ -36,8 +36,11 @@ import ( logger "d7y.io/dragonfly/v2/internal/dflog" ) +// defaultValidityDuration is default validity duration of certificate. +const defaultValidityDuration = 365 * 24 * time.Hour + func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.CertificateRequest) (*securityv1.CertificateResponse, error) { - if s.cert == nil { + if s.selfSignedCert == nil { return nil, status.Errorf(codes.Unavailable, "ca is missing for this manager instance") } @@ -86,7 +89,7 @@ func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.Certifica now := time.Now() duration := time.Duration(req.ValidityDuration) * time.Second if duration == 0 { - duration = time.Hour + duration = defaultValidityDuration } logger.Infof("valid csr: %#v", csr.Subject) @@ -102,18 +105,18 @@ func (s *Server) IssueCertificate(ctx context.Context, req *securityv1.Certifica ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, } - cert, err := x509.CreateCertificate(rand.Reader, &template, s.x509Cert, csr.PublicKey, s.cert.PrivateKey) + cert, err := x509.CreateCertificate(rand.Reader, &template, s.selfSignedCert.X509Cert, csr.PublicKey, s.selfSignedCert.TLSCert.PrivateKey) if err != nil { return nil, fmt.Errorf("failed to generate certificate, error: %s", err) } - // Encode into PEM format. + // Build the certificate chain. var certPEM bytes.Buffer if err = pem.Encode(&certPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { return nil, err } return &securityv1.CertificateResponse{ - CertificateChain: append([]string{certPEM.String()}, s.certChain...), + CertificateChain: append([]string{certPEM.String()}, s.selfSignedCert.PEMCertChain...), }, nil } diff --git a/manager/rpcserver/cert_test.go b/manager/rpcserver/cert_test.go index c1a18fb8e..84bdeabc2 100644 --- a/manager/rpcserver/cert_test.go +++ b/manager/rpcserver/cert_test.go @@ -89,7 +89,7 @@ func TestIssueCertificate(t *testing.T) { DB: &gorm.DB{}, RDB: &redis.Client{}, }, - nil, nil, nil, nil, WithCertificate(&ca)) + nil, nil, nil, nil, WithSelfSignedCert(&ca)) require.Nilf(err, "newServer should be ok") ctx := peer.NewContext( @@ -110,7 +110,7 @@ func TestIssueCertificate(t *testing.T) { assert.Nilf(err, "IssueCertificate should be ok") assert.NotNilf(resp, "IssueCertificate should not be nil") - assert.Equal(len(resp.CertificateChain), len(server.certChain)+1) + assert.Equal(len(resp.CertificateChain), 2) cert := readCert(resp.CertificateChain[0]) assert.Equal(len(cert.IPAddresses), 1) diff --git a/manager/rpcserver/rpcserver.go b/manager/rpcserver/rpcserver.go index 756240481..f6d6b1493 100644 --- a/manager/rpcserver/rpcserver.go +++ b/manager/rpcserver/rpcserver.go @@ -50,6 +50,18 @@ import ( managerserver "d7y.io/dragonfly/v2/pkg/rpc/manager/server" ) +// SelfSignedCert is self signed certificate. +type SelfSignedCert struct { + // TLSCert is certificate of tls. + TLSCert *tls.Certificate + + // X509Cert is certificate of x509. + X509Cert *x509.Certificate + + // PEMCertChain is certificate chain of pem. + PEMCertChain []string +} + // Server is grpc server. type Server struct { // Manager configuration. @@ -76,39 +88,35 @@ type Server struct { // serverOptions is server options of grpc. serverOptions []grpc.ServerOption - // cert certificates to sign certificates. - cert *tls.Certificate - - // x509Cert certificates to sign certificates. - x509Cert *x509.Certificate - - // certChain is PEM-encoded certificate chain. - certChain []string + // selfSignedCert is self signed certificate. + selfSignedCert *SelfSignedCert } // Option is a functional option for rpc server. type Option func(s *Server) error -// WithCertificate set the root tls certificate, x509 certificate and PEM-encoded certificate chain. -func WithCertificate(cert *tls.Certificate) Option { +// WithCertificate set the self signed certificate for server. +func WithSelfSignedCert(tlsCert *tls.Certificate) Option { return func(s *Server) error { - s.cert = cert - - // Parse x509 certificate from tls certificate. - var err error - s.x509Cert, err = x509.ParseCertificate(cert.Certificate[0]) + x509CACert, err := x509.ParseCertificate(tlsCert.Certificate[0]) if err != nil { return err } - // Initialize PEM-encoded certificate chain from tls certificate. - for _, cert := range cert.Certificate { - var certChainPEM bytes.Buffer - if err = pem.Encode(&certChainPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { + var pemCertChain []string + for _, cert := range tlsCert.Certificate { + var certPEM bytes.Buffer + if err := pem.Encode(&certPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { return err } - s.certChain = append(s.certChain, certChainPEM.String()) + pemCertChain = append(pemCertChain, certPEM.String()) + } + + s.selfSignedCert = &SelfSignedCert{ + TLSCert: tlsCert, + X509Cert: x509CACert, + PEMCertChain: pemCertChain, } return nil diff --git a/pkg/cache/certify.go b/pkg/cache/certify.go index cd5224a72..fcb2559b7 100644 --- a/pkg/cache/certify.go +++ b/pkg/cache/certify.go @@ -23,18 +23,23 @@ import ( "github.com/johanbrandhorst/certify" ) +const ( + // CertifyCacheDirName is dir name of certify cache. + CertifyCacheDirName = "certs" +) + type certifyCache struct { caches []certify.Cache } // NewCertifyMutliCache returns a certify.Cache with multiple caches // Such as, cache.NewCertifyMutliCache(certify.NewMemCache(), certify.DirCache("certs")) -// This multiple cache will get certs from mem cache first, then dir cache to avoid read from filesystem every times +// This multiple cache will get certs from mem cache first, then dir cache to avoid read from filesystem every times. func NewCertifyMutliCache(caches ...certify.Cache) certify.Cache { return &certifyCache{caches: caches} } -// Get gets cert from cache one by one, if found, puts it back to all previous cachs +// Get gets cert from cache one by one, if found, puts it back to all previous caches. func (c *certifyCache) Get(ctx context.Context, key string) (cert *tls.Certificate, err error) { var foundCacheIdx int = -1 for i, cache := range c.caches { @@ -55,7 +60,7 @@ func (c *certifyCache) Get(ctx context.Context, key string) (cert *tls.Certifica return nil, certify.ErrCacheMiss } -// Put puts cert to all caches +// Put puts cert to all caches. func (c *certifyCache) Put(ctx context.Context, key string, cert *tls.Certificate) error { for _, cache := range c.caches { if err := cache.Put(ctx, key, cert); err != nil { @@ -65,7 +70,7 @@ func (c *certifyCache) Put(ctx context.Context, key string, cert *tls.Certificat return nil } -// Delete deletes cert from all caches +// Delete deletes cert from all caches. func (c *certifyCache) Delete(ctx context.Context, key string) error { for _, cache := range c.caches { if err := cache.Delete(ctx, key); err != nil { diff --git a/pkg/issuer/dragonfly_manager.go b/pkg/issuer/dragonfly_manager.go new file mode 100644 index 000000000..ec9d176d4 --- /dev/null +++ b/pkg/issuer/dragonfly_manager.go @@ -0,0 +1,165 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package issuer + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "net/url" + "time" + + "github.com/johanbrandhorst/certify" + + "d7y.io/dragonfly/v2/pkg/net/ip" +) + +var ( + // defaultSubjectCommonName is default common name of subject. + defaultSubjectCommonName = "manager" + + // defaultSubjectOrganization is default organization of subject. + defaultSubjectOrganization = []string{"Dragonfly"} + + // defaultSubjectOrganizationalUnit is default organizational unit of subject. + defaultSubjectOrganizationalUnit = []string{"Manager"} + + // defaultIPAddresses is default ip addresses of certificate. + defaultIPAddresses = []net.IP{net.ParseIP(ip.IPv4)} + + // defaultDNSNames is default dns names of certificate. + defaultDNSNames = []string{"dragonfly-manager", "dragonfly-manager.dragonfly-system.svc"} + + // defaultValidityDuration is default validity duration of certificate. + defaultValidityDuration = 10 * 365 * 24 * time.Hour +) + +// GC provides issuer function. +type dragonflyManagerIssuer struct { + tlsCACert *tls.Certificate + dnsNames []string + emailAddresses []string + ipAddresses []net.IP + uris []*url.URL + validityDuration time.Duration +} + +// Option is a functional option for configuring the dragonflyManagerIssuer. +type Option func(i *dragonflyManagerIssuer) + +// WithDNSNames set the dnsNames for dragonflyManagerIssuer. +func WithDNSNames(dnsNames []string) Option { + return func(i *dragonflyManagerIssuer) { + i.dnsNames = dnsNames + } +} + +// WithEmailAddresses set the emailAddresses for dragonflyManagerIssuer. +func WithEmailAddresses(emailAddrs []string) Option { + return func(i *dragonflyManagerIssuer) { + i.emailAddresses = emailAddrs + } +} + +// WithIPAddresses set the ipAddresses for dragonflyManagerIssuer. +func WithIPAddresses(ipAddrs []net.IP) Option { + return func(i *dragonflyManagerIssuer) { + i.ipAddresses = ipAddrs + } +} + +// WithURIs set the uris for dragonflyManagerIssuer. +func WithURIs(uris []*url.URL) Option { + return func(i *dragonflyManagerIssuer) { + i.uris = uris + } +} + +// WithValidityDuration set the validityDuration for dragonflyManagerIssuer. +func WithValidityDuration(d time.Duration) Option { + return func(i *dragonflyManagerIssuer) { + i.validityDuration = d + } +} + +// NewDragonflyManagerIssuer returns a new certify.Issuer instence. +func NewDragonflyManagerIssuer(tlsCACert *tls.Certificate, opts ...Option) certify.Issuer { + i := &dragonflyManagerIssuer{ + tlsCACert: tlsCACert, + dnsNames: defaultDNSNames, + ipAddresses: defaultIPAddresses, + validityDuration: defaultValidityDuration, + } + + for _, opt := range opts { + opt(i) + } + + return i +} + +// Issue returns tls Certificate of issuing. +func (i *dragonflyManagerIssuer) Issue(ctx context.Context, commonName string, certConfig *certify.CertConfig) (*tls.Certificate, error) { + x509CACert, err := x509.ParseCertificate(i.tlsCACert.Certificate[0]) + if err != nil { + return nil, err + } + + pk, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + serial, err := rand.Int(rand.Reader, (&big.Int{}).Exp(big.NewInt(2), big.NewInt(159), nil)) + if err != nil { + return nil, err + } + + now := time.Now() + template := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: defaultSubjectCommonName, + Organization: defaultSubjectOrganization, + OrganizationalUnit: defaultSubjectOrganizationalUnit, + }, + DNSNames: i.dnsNames, + EmailAddresses: i.emailAddresses, + IPAddresses: i.ipAddresses, + URIs: i.uris, + NotBefore: now.Add(-10 * time.Minute).UTC(), + NotAfter: now.Add(i.validityDuration).UTC(), + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageDataEncipherment | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + cert, err := x509.CreateCertificate(rand.Reader, &template, x509CACert, &pk.PublicKey, i.tlsCACert.PrivateKey) + if err != nil { + return nil, err + } + + return &tls.Certificate{ + Certificate: append([][]byte{cert}, i.tlsCACert.Certificate...), + PrivateKey: pk, + }, nil +} diff --git a/pkg/rpc/credential.go b/pkg/rpc/credential.go new file mode 100644 index 000000000..e1faab083 --- /dev/null +++ b/pkg/rpc/credential.go @@ -0,0 +1,50 @@ +/* + * Copyright 2022 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rpc + +import ( + "crypto/tls" + "crypto/x509" + "errors" + + "github.com/johanbrandhorst/certify" + "google.golang.org/grpc/credentials" +) + +// NewServerCredentialsByCertify returns server transport credentials by certify. +func NewServerCredentialsByCertify(tlsVerify bool, tlsCACert *tls.Certificate, certifyClient *certify.Certify) (credentials.TransportCredentials, error) { + if !tlsVerify { + return credentials.NewTLS(&tls.Config{ + GetCertificate: certifyClient.GetCertificate, + }), nil + } + + certPool := x509.NewCertPool() + for _, cert := range tlsCACert.Certificate { + if !certPool.AppendCertsFromPEM(cert) { + return nil, errors.New("invalid CA Cert") + } + } + + tlsConfig := &tls.Config{ + GetCertificate: certifyClient.GetCertificate, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: certPool, + } + + return credentials.NewTLS(tlsConfig), nil +}