From 17fb1b287fea81a628e14f51719224ac8885caf3 Mon Sep 17 00:00:00 2001 From: Phil Porada Date: Mon, 24 Apr 2023 16:28:05 -0400 Subject: [PATCH] cmd: Export prometheus metrics for TLS cert notBefore and notAfter fields (#6836) Export new prometheus metrics for the `notBefore` and `notAfter` fields to track internal certificate validity periods when calling the `Load()` method for a `*tls.Config`. Each metric is labeled with the `serial` field. ``` tlsconfig_notafter_seconds{serial="2152072875247971686"} 1.664821961e+09 tlsconfig_notbefore_seconds{serial="2152072875247971686"} 1.664821960e+09 ``` Fixes https://github.com/letsencrypt/boulder/issues/6829 --- cmd/admin-revoker/main.go | 3 +- cmd/akamai-purger/main.go | 2 +- cmd/bad-key-revoker/main.go | 2 +- cmd/boulder-ca/main.go | 2 +- cmd/boulder-publisher/main.go | 3 +- cmd/boulder-ra/main.go | 2 +- cmd/boulder-sa/main.go | 2 +- cmd/boulder-va/main.go | 2 +- cmd/boulder-wfe2/main.go | 2 +- cmd/config.go | 48 ++++++++++++++++++- cmd/config_test.go | 4 +- cmd/crl-storer/main.go | 6 +-- cmd/crl-updater/main.go | 6 +-- cmd/expiration-mailer/main.go | 2 +- cmd/nonce-service/main.go | 2 +- cmd/ocsp-responder/main.go | 2 +- cmd/orphan-finder/main.go | 3 +- cmd/rocsp-tool/client_test.go | 2 +- cmd/rocsp-tool/main.go | 2 +- rocsp/config/rocsp_config.go | 4 +- rocsp/rocsp_test.go | 2 +- test/health-checker/main.go | 3 +- .../akamai_purger_drain_queue_test.go | 3 +- 23 files changed, 78 insertions(+), 31 deletions(-) diff --git a/cmd/admin-revoker/main.go b/cmd/admin-revoker/main.go index 94a053bbb..375f779f0 100644 --- a/cmd/admin-revoker/main.go +++ b/cmd/admin-revoker/main.go @@ -100,7 +100,8 @@ type revoker struct { func newRevoker(c Config) *revoker { logger := cmd.NewLogger(c.Syslog) - tlsConfig, err := c.Revoker.TLS.Load() + // TODO(#6840) Rework admin-revoker to export prometheus metrics. + tlsConfig, err := c.Revoker.TLS.Load(metrics.NoopRegisterer) cmd.FailOnError(err, "TLS config") clk := cmd.Clock() diff --git a/cmd/akamai-purger/main.go b/cmd/akamai-purger/main.go index 9f81c8262..f0bc3788c 100644 --- a/cmd/akamai-purger/main.go +++ b/cmd/akamai-purger/main.go @@ -366,7 +366,7 @@ func manualPurge(purgeClient *akamai.CachePurgeClient, tag, tagFile string) { func daemon(c Config, ap *akamaiPurger, logger blog.Logger, scope prometheus.Registerer) { clk := cmd.Clock() - tlsConfig, err := c.AkamaiPurger.TLS.Load() + tlsConfig, err := c.AkamaiPurger.TLS.Load(scope) cmd.FailOnError(err, "tlsConfig config") stop, stopped := make(chan bool, 1), make(chan bool, 1) diff --git a/cmd/bad-key-revoker/main.go b/cmd/bad-key-revoker/main.go index 527d4d36a..8fc67e0c7 100644 --- a/cmd/bad-key-revoker/main.go +++ b/cmd/bad-key-revoker/main.go @@ -450,7 +450,7 @@ func main() { dbMap, err := sa.InitWrappedDb(config.BadKeyRevoker.DB, scope, logger) cmd.FailOnError(err, "While initializing dbMap") - tlsConfig, err := config.BadKeyRevoker.TLS.Load() + tlsConfig, err := config.BadKeyRevoker.TLS.Load(scope) cmd.FailOnError(err, "TLS config") conn, err := bgrpc.ClientSetup(config.BadKeyRevoker.RAService, tlsConfig, scope, clk) diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index 60ce83f20..b9e7125ec 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -214,7 +214,7 @@ func main() { boulderIssuers, err = loadBoulderIssuers(c.CA.Issuance.Profile, c.CA.Issuance.Issuers, c.CA.Issuance.IgnoredLints) cmd.FailOnError(err, "Couldn't load issuers") - tlsConfig, err := c.CA.TLS.Load() + tlsConfig, err := c.CA.TLS.Load(scope) cmd.FailOnError(err, "TLS config") clk := cmd.Clock() diff --git a/cmd/boulder-publisher/main.go b/cmd/boulder-publisher/main.go index 17ddf5964..435c0b7f0 100644 --- a/cmd/boulder-publisher/main.go +++ b/cmd/boulder-publisher/main.go @@ -65,7 +65,6 @@ func main() { if c.Publisher.UserAgent == "" { c.Publisher.UserAgent = "certificate-transparency-go/1.0" } - scope, logger, oTelShutdown := cmd.StatsAndLogging(c.Syslog, c.OpenTelemetry, c.Publisher.DebugAddr) defer oTelShutdown(context.Background()) defer logger.AuditPanic() @@ -88,7 +87,7 @@ func main() { bundles[id] = publisher.GetCTBundleForChain(chain) } - tlsConfig, err := c.Publisher.TLS.Load() + tlsConfig, err := c.Publisher.TLS.Load(scope) cmd.FailOnError(err, "TLS config") clk := cmd.Clock() diff --git a/cmd/boulder-ra/main.go b/cmd/boulder-ra/main.go index f6f85d31e..e0018519b 100644 --- a/cmd/boulder-ra/main.go +++ b/cmd/boulder-ra/main.go @@ -140,7 +140,7 @@ func main() { err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) cmd.FailOnError(err, "Couldn't load hostname policy file") - tlsConfig, err := c.RA.TLS.Load() + tlsConfig, err := c.RA.TLS.Load(scope) cmd.FailOnError(err, "TLS config") clk := cmd.Clock() diff --git a/cmd/boulder-sa/main.go b/cmd/boulder-sa/main.go index 3938e3fbf..51d5d5efa 100644 --- a/cmd/boulder-sa/main.go +++ b/cmd/boulder-sa/main.go @@ -84,7 +84,7 @@ func main() { parallel = 1 } - tls, err := c.SA.TLS.Load() + tls, err := c.SA.TLS.Load(scope) cmd.FailOnError(err, "TLS config") saroi, err := sa.NewSQLStorageAuthorityRO( diff --git a/cmd/boulder-va/main.go b/cmd/boulder-va/main.go index ebc83fa14..d67b7823d 100644 --- a/cmd/boulder-va/main.go +++ b/cmd/boulder-va/main.go @@ -120,7 +120,7 @@ func main() { logger) } - tlsConfig, err := c.VA.TLS.Load() + tlsConfig, err := c.VA.TLS.Load(scope) cmd.FailOnError(err, "tlsConfig config") var remotes []va.RemoteVA diff --git a/cmd/boulder-wfe2/main.go b/cmd/boulder-wfe2/main.go index 615005a15..d9d4b3dae 100644 --- a/cmd/boulder-wfe2/main.go +++ b/cmd/boulder-wfe2/main.go @@ -311,7 +311,7 @@ func loadChain(certFiles []string) (*issuance.Certificate, []byte, error) { } func setupWFE(c Config, scope prometheus.Registerer, clk clock.Clock) (rapb.RegistrationAuthorityClient, sapb.StorageAuthorityReadOnlyClient, nonce.Getter, map[string]nonce.Redeemer, nonce.Redeemer, string) { - tlsConfig, err := c.WFE.TLS.Load() + tlsConfig, err := c.WFE.TLS.Load(scope) cmd.FailOnError(err, "TLS config") raConn, err := bgrpc.ClientSetup(c.WFE.RAService, tlsConfig, scope, clk) diff --git a/cmd/config.go b/cmd/config.go index 73a111178..2446e7b75 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/go-sql-driver/mysql" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc/resolver" "github.com/letsencrypt/boulder/config" @@ -139,8 +140,9 @@ type TLSConfig struct { } // Load reads and parses the certificates and key listed in the TLSConfig, and -// returns a *tls.Config suitable for either client or server use. -func (t *TLSConfig) Load() (*tls.Config, error) { +// returns a *tls.Config suitable for either client or server use. Prometheus +// metrics for various certificate fields will be exported. +func (t *TLSConfig) Load(scope prometheus.Registerer) (*tls.Config, error) { if t == nil { return nil, fmt.Errorf("nil TLS section in config") } @@ -166,6 +168,48 @@ func (t *TLSConfig) Load() (*tls.Config, error) { return nil, fmt.Errorf("loading key pair from %q and %q: %s", *t.CertFile, *t.KeyFile, err) } + + tlsNotBefore := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "tlsconfig_notbefore_seconds", + Help: "TLS certificate NotBefore field expressed as Unix epoch time", + }, + []string{"serial"}) + err = scope.Register(tlsNotBefore) + if err != nil { + are := prometheus.AlreadyRegisteredError{} + if errors.As(err, &are) { + tlsNotBefore = are.ExistingCollector.(*prometheus.GaugeVec) + } else { + return nil, err + } + } + + tlsNotAfter := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "tlsconfig_notafter_seconds", + Help: "TLS certificate NotAfter field expressed as Unix epoch time", + }, + []string{"serial"}) + err = scope.Register(tlsNotAfter) + if err != nil { + are := prometheus.AlreadyRegisteredError{} + if errors.As(err, &are) { + tlsNotAfter = are.ExistingCollector.(*prometheus.GaugeVec) + } else { + return nil, err + } + } + + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, err + } + + serial := leaf.SerialNumber.String() + tlsNotBefore.WithLabelValues(serial).Set(float64(leaf.NotBefore.Unix())) + tlsNotAfter.WithLabelValues(serial).Set(float64(leaf.NotAfter.Unix())) + return &tls.Config{ RootCAs: rootCAs, ClientCAs: rootCAs, diff --git a/cmd/config_test.go b/cmd/config_test.go index 6988e1e77..4a18fda4d 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/letsencrypt/boulder/metrics" "github.com/letsencrypt/boulder/test" ) @@ -54,6 +55,7 @@ func TestTLSConfigLoad(t *testing.T) { cert := "testdata/cert.pem" key := "testdata/key.pem" caCert := "testdata/minica.pem" + testCases := []struct { TLSConfig want string @@ -86,7 +88,7 @@ func TestTLSConfigLoad(t *testing.T) { title[2] = *tc.CACertFile } t.Run(strings.Join(title[:], "_"), func(t *testing.T) { - _, err := tc.TLSConfig.Load() + _, err := tc.TLSConfig.Load(metrics.NoopRegisterer) if err == nil { t.Errorf("got no error") } diff --git a/cmd/crl-storer/main.go b/cmd/crl-storer/main.go index 1f75fc305..c7af47697 100644 --- a/cmd/crl-storer/main.go +++ b/cmd/crl-storer/main.go @@ -82,15 +82,15 @@ func main() { err = features.Set(c.CRLStorer.Features) cmd.FailOnError(err, "Failed to set feature flags") - tlsConfig, err := c.CRLStorer.TLS.Load() - cmd.FailOnError(err, "TLS config") - scope, logger, oTelShutdown := cmd.StatsAndLogging(c.Syslog, c.OpenTelemetry, c.CRLStorer.DebugAddr) defer oTelShutdown(context.Background()) defer logger.AuditPanic() logger.Info(cmd.VersionString()) clk := cmd.Clock() + tlsConfig, err := c.CRLStorer.TLS.Load(scope) + cmd.FailOnError(err, "TLS config") + issuers := make([]*issuance.Certificate, 0, len(c.CRLStorer.IssuerCerts)) for _, filepath := range c.CRLStorer.IssuerCerts { cert, err := issuance.LoadCertificate(filepath) diff --git a/cmd/crl-updater/main.go b/cmd/crl-updater/main.go index bb08c80d9..9471931ce 100644 --- a/cmd/crl-updater/main.go +++ b/cmd/crl-updater/main.go @@ -115,15 +115,15 @@ func main() { err = features.Set(c.CRLUpdater.Features) cmd.FailOnError(err, "Failed to set feature flags") - tlsConfig, err := c.CRLUpdater.TLS.Load() - cmd.FailOnError(err, "TLS config") - scope, logger, oTelShutdown := cmd.StatsAndLogging(c.Syslog, c.OpenTelemetry, c.CRLUpdater.DebugAddr) defer oTelShutdown(context.Background()) defer logger.AuditPanic() logger.Info(cmd.VersionString()) clk := cmd.Clock() + tlsConfig, err := c.CRLUpdater.TLS.Load(scope) + cmd.FailOnError(err, "TLS config") + issuers := make([]*issuance.Certificate, 0, len(c.CRLUpdater.IssuerCerts)) for _, filepath := range c.CRLUpdater.IssuerCerts { cert, err := issuance.LoadCertificate(filepath) diff --git a/cmd/expiration-mailer/main.go b/cmd/expiration-mailer/main.go index c1d239343..1cc24dd94 100644 --- a/cmd/expiration-mailer/main.go +++ b/cmd/expiration-mailer/main.go @@ -825,7 +825,7 @@ func main() { dbMap, err := sa.InitWrappedDb(c.Mailer.DB, scope, logger) cmd.FailOnError(err, "While initializing dbMap") - tlsConfig, err := c.Mailer.TLS.Load() + tlsConfig, err := c.Mailer.TLS.Load(scope) cmd.FailOnError(err, "TLS config") clk := cmd.Clock() diff --git a/cmd/nonce-service/main.go b/cmd/nonce-service/main.go index 0323362a2..92fef9816 100644 --- a/cmd/nonce-service/main.go +++ b/cmd/nonce-service/main.go @@ -108,7 +108,7 @@ func main() { ns, err := nonce.NewNonceService(scope, c.NonceService.MaxUsed, c.NonceService.NoncePrefix) cmd.FailOnError(err, "Failed to initialize nonce service") - tlsConfig, err := c.NonceService.TLS.Load() + tlsConfig, err := c.NonceService.TLS.Load(scope) cmd.FailOnError(err, "tlsConfig config") nonceServer := nonce.NewServer(ns) diff --git a/cmd/ocsp-responder/main.go b/cmd/ocsp-responder/main.go index c76f87eae..2254dc26f 100644 --- a/cmd/ocsp-responder/main.go +++ b/cmd/ocsp-responder/main.go @@ -163,7 +163,7 @@ as generated by Boulder's ceremony command. liveSigningPeriod = 60 * time.Hour } - tlsConfig, err := c.OCSPResponder.TLS.Load() + tlsConfig, err := c.OCSPResponder.TLS.Load(scope) cmd.FailOnError(err, "TLS config") raConn, err := bgrpc.ClientSetup(c.OCSPResponder.RAService, tlsConfig, scope, clk) diff --git a/cmd/orphan-finder/main.go b/cmd/orphan-finder/main.go index 512512bd0..2823d8fb7 100644 --- a/cmd/orphan-finder/main.go +++ b/cmd/orphan-finder/main.go @@ -174,7 +174,8 @@ func newOrphanFinder(configFile string) *orphanFinder { cmd.FailOnError(err, "Failed to set feature flags") logger := cmd.NewLogger(conf.Syslog) - tlsConfig, err := conf.TLS.Load() + // TODO(#6840) Rework orphan-finder to export prometheus metrics. + tlsConfig, err := conf.TLS.Load(metrics.NoopRegisterer) cmd.FailOnError(err, "TLS config") saConn, err := bgrpc.ClientSetup(conf.SAService, tlsConfig, metrics.NoopRegisterer, cmd.Clock()) diff --git a/cmd/rocsp-tool/client_test.go b/cmd/rocsp-tool/client_test.go index ea80daf9e..8365e44b6 100644 --- a/cmd/rocsp-tool/client_test.go +++ b/cmd/rocsp-tool/client_test.go @@ -31,7 +31,7 @@ func makeClient() (*rocsp.RWClient, clock.Clock) { CertFile: &CertFile, KeyFile: &KeyFile, } - tlsConfig2, err := tlsConfig.Load() + tlsConfig2, err := tlsConfig.Load(metrics.NoopRegisterer) if err != nil { panic(err) } diff --git a/cmd/rocsp-tool/main.go b/cmd/rocsp-tool/main.go index ac4f01721..6f537121a 100644 --- a/cmd/rocsp-tool/main.go +++ b/cmd/rocsp-tool/main.go @@ -249,7 +249,7 @@ func helpExit() { } func configureOCSPGenerator(tlsConf cmd.TLSConfig, grpcConf cmd.GRPCClientConfig, clk clock.Clock, scope prometheus.Registerer) (capb.OCSPGeneratorClient, error) { - tlsConfig, err := tlsConf.Load() + tlsConfig, err := tlsConf.Load(scope) if err != nil { return nil, fmt.Errorf("loading TLS config: %w", err) } diff --git a/rocsp/config/rocsp_config.go b/rocsp/config/rocsp_config.go index 93edff0ca..0492175c7 100644 --- a/rocsp/config/rocsp_config.go +++ b/rocsp/config/rocsp_config.go @@ -102,7 +102,7 @@ func MakeClient(c *RedisConfig, clk clock.Clock, stats prometheus.Registerer) (* return nil, fmt.Errorf("loading password: %w", err) } - tlsConfig, err := c.TLS.Load() + tlsConfig, err := c.TLS.Load(stats) if err != nil { return nil, fmt.Errorf("loading TLS config: %w", err) } @@ -141,7 +141,7 @@ func MakeReadClient(c *RedisConfig, clk clock.Clock, stats prometheus.Registerer return nil, fmt.Errorf("loading password: %w", err) } - tlsConfig, err := c.TLS.Load() + tlsConfig, err := c.TLS.Load(stats) if err != nil { return nil, fmt.Errorf("loading TLS config: %w", err) } diff --git a/rocsp/rocsp_test.go b/rocsp/rocsp_test.go index c5b3f2f11..ba2e2aef5 100644 --- a/rocsp/rocsp_test.go +++ b/rocsp/rocsp_test.go @@ -24,7 +24,7 @@ func makeClient() (*RWClient, clock.Clock) { CertFile: &CertFile, KeyFile: &KeyFile, } - tlsConfig2, err := tlsConfig.Load() + tlsConfig2, err := tlsConfig.Load(metrics.NoopRegisterer) if err != nil { panic(err) } diff --git a/test/health-checker/main.go b/test/health-checker/main.go index 7ae41cfd6..74bb884c0 100644 --- a/test/health-checker/main.go +++ b/test/health-checker/main.go @@ -42,7 +42,7 @@ func main() { c.GRPC.ServerAddress = *serverAddr } - tlsConfig, err := c.TLS.Load() + tlsConfig, err := c.TLS.Load(metrics.NoopRegisterer) cmd.FailOnError(err, "failed to load TLS credentials") // GRPC connection prerequisites. @@ -75,7 +75,6 @@ func main() { Service: "", } resp, err := client.Check(ctx2, req) - if err != nil { fmt.Fprintf(os.Stderr, "got error connecting to health service %s: %s\n", *serverAddr, err) } else if resp.Status == healthpb.HealthCheckResponse_SERVING { diff --git a/test/integration/akamai_purger_drain_queue_test.go b/test/integration/akamai_purger_drain_queue_test.go index 8a764ae00..5689496a9 100644 --- a/test/integration/akamai_purger_drain_queue_test.go +++ b/test/integration/akamai_purger_drain_queue_test.go @@ -15,6 +15,7 @@ import ( akamaipb "github.com/letsencrypt/boulder/akamai/proto" "github.com/letsencrypt/boulder/cmd" bcreds "github.com/letsencrypt/boulder/grpc/creds" + "github.com/letsencrypt/boulder/metrics" "github.com/letsencrypt/boulder/test" "google.golang.org/grpc" "google.golang.org/grpc/balancer/roundrobin" @@ -42,7 +43,7 @@ func setup() (*exec.Cmd, *bytes.Buffer, akamaipb.AkamaiPurgerClient, error) { CACertFile: s("test/grpc-creds/minica.pem"), CertFile: s("test/grpc-creds/ra.boulder/cert.pem"), KeyFile: s("test/grpc-creds/ra.boulder/key.pem"), - }).Load() + }).Load(metrics.NoopRegisterer) if err != nil { sigterm() return nil, nil, nil, err