package publisher import ( "context" "crypto/ecdsa" "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/asn1" "encoding/base64" "encoding/json" "errors" "fmt" "math/big" "net/http" "net/url" "strings" "sync" "time" ct "github.com/google/certificate-transparency-go" ctClient "github.com/google/certificate-transparency-go/client" "github.com/google/certificate-transparency-go/jsonclient" cttls "github.com/google/certificate-transparency-go/tls" "github.com/prometheus/client_golang/prometheus" "github.com/letsencrypt/boulder/core" "github.com/letsencrypt/boulder/issuance" blog "github.com/letsencrypt/boulder/log" "github.com/letsencrypt/boulder/metrics" pubpb "github.com/letsencrypt/boulder/publisher/proto" ) // Log contains the CT client for a particular CT log type Log struct { logID string uri string client *ctClient.LogClient } // cacheKey is a comparable type for use as a key within a logCache. It holds // both the log URI and its log_id (base64 encoding of its pubkey), so that // the cache won't interfere if the RA decides that a log's URI or pubkey has // changed. type cacheKey struct { uri string pubkey string } // logCache contains a cache of *Log's that are constructed as required by // `SubmitToSingleCT` type logCache struct { sync.RWMutex logs map[cacheKey]*Log } // AddLog adds a *Log to the cache by constructing the statName, client and // verifier for the given uri & base64 public key. func (c *logCache) AddLog(uri, b64PK, userAgent string, logger blog.Logger) (*Log, error) { // Lock the mutex for reading to check the cache c.RLock() log, present := c.logs[cacheKey{uri, b64PK}] c.RUnlock() // If we have already added this log, give it back if present { return log, nil } // Lock the mutex for writing to add to the cache c.Lock() defer c.Unlock() // Construct a Log, add it to the cache, and return it to the caller log, err := NewLog(uri, b64PK, userAgent, logger) if err != nil { return nil, err } c.logs[cacheKey{uri, b64PK}] = log return log, nil } // Len returns the number of logs in the logCache func (c *logCache) Len() int { c.RLock() defer c.RUnlock() return len(c.logs) } type logAdaptor struct { blog.Logger } func (la logAdaptor) Printf(s string, args ...interface{}) { la.Logger.Infof(s, args...) } // NewLog returns an initialized Log struct func NewLog(uri, b64PK, userAgent string, logger blog.Logger) (*Log, error) { url, err := url.Parse(uri) if err != nil { return nil, err } url.Path = strings.TrimSuffix(url.Path, "/") derPK, err := base64.StdEncoding.DecodeString(b64PK) if err != nil { return nil, err } opts := jsonclient.Options{ Logger: logAdaptor{logger}, PublicKeyDER: derPK, UserAgent: userAgent, } httpClient := &http.Client{ // We set the HTTP client timeout to about half of what we expect // the gRPC timeout to be set to. This allows us to retry the // request at least twice in the case where the server we are // talking to is simply hanging indefinitely. Timeout: time.Minute*2 + time.Second*30, // We provide a new Transport for each Client so that different logs don't // share a connection pool. This shouldn't matter, but we occasionally see a // strange bug where submission to all logs hangs for about fifteen minutes. // One possibility is that there is a strange bug in the locking on // connection pools (possibly triggered by timed-out TCP connections). If // that's the case, separate connection pools should prevent cross-log impact. // We set some fields like TLSHandshakeTimeout to the values from // DefaultTransport because the zero value for these fields means // "unlimited," which would be bad. Transport: &http.Transport{ MaxIdleConns: http.DefaultTransport.(*http.Transport).MaxIdleConns, MaxIdleConnsPerHost: http.DefaultTransport.(*http.Transport).MaxIdleConns, IdleConnTimeout: http.DefaultTransport.(*http.Transport).IdleConnTimeout, TLSHandshakeTimeout: http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout, // In Boulder Issue 3821[0] we found that HTTP/2 support was causing hard // to diagnose intermittent freezes in CT submission. Disabling HTTP/2 with // an environment variable resolved the freezes but is not a stable fix. // // Per the Go `http` package docs we can make this change persistent by // changing the `http.Transport` config: // "Programs that must disable HTTP/2 can do so by setting // Transport.TLSNextProto (for clients) or Server.TLSNextProto (for // servers) to a non-nil, empty map" // // [0]: https://github.com/letsencrypt/boulder/issues/3821 TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, }, } client, err := ctClient.New(url.String(), httpClient, opts) if err != nil { return nil, fmt.Errorf("making CT client: %s", err) } return &Log{ logID: b64PK, uri: url.String(), client: client, }, nil } type ctSubmissionRequest struct { Chain []string `json:"chain"` } type pubMetrics struct { submissionLatency *prometheus.HistogramVec probeLatency *prometheus.HistogramVec errorCount *prometheus.CounterVec } func initMetrics(stats prometheus.Registerer) *pubMetrics { submissionLatency := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "ct_submission_time_seconds", Help: "Time taken to submit a certificate to a CT log", Buckets: metrics.InternetFacingBuckets, }, []string{"log", "type", "status", "http_status"}, ) stats.MustRegister(submissionLatency) probeLatency := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "ct_probe_time_seconds", Help: "Time taken to probe a CT log", Buckets: metrics.InternetFacingBuckets, }, []string{"log", "status"}, ) stats.MustRegister(probeLatency) errorCount := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "ct_errors_count", Help: "Count of errors by type", }, []string{"log", "type"}, ) stats.MustRegister(errorCount) return &pubMetrics{submissionLatency, probeLatency, errorCount} } // Impl defines a Publisher type Impl struct { pubpb.UnsafePublisherServer log blog.Logger userAgent string issuerBundles map[issuance.NameID][]ct.ASN1Cert ctLogsCache logCache metrics *pubMetrics } var _ pubpb.PublisherServer = (*Impl)(nil) // New creates a Publisher that will submit certificates // to requested CT logs func New( bundles map[issuance.NameID][]ct.ASN1Cert, userAgent string, logger blog.Logger, stats prometheus.Registerer, ) *Impl { return &Impl{ issuerBundles: bundles, userAgent: userAgent, ctLogsCache: logCache{ logs: make(map[cacheKey]*Log), }, log: logger, metrics: initMetrics(stats), } } // SubmitToSingleCTWithResult will submit the certificate represented by certDER // to the CT log specified by log URL and public key (base64) and return the SCT // to the caller. func (pub *Impl) SubmitToSingleCTWithResult(ctx context.Context, req *pubpb.Request) (*pubpb.Result, error) { if core.IsAnyNilOrZero(req.Der, req.LogURL, req.LogPublicKey, req.Kind) { return nil, errors.New("incomplete gRPC request message") } cert, err := x509.ParseCertificate(req.Der) if err != nil { pub.log.AuditErrf("Failed to parse certificate: %s", err) return nil, err } chain := []ct.ASN1Cert{{Data: req.Der}} id := issuance.IssuerNameID(cert) issuerBundle, ok := pub.issuerBundles[id] if !ok { err := fmt.Errorf("No issuerBundle matching issuerNameID: %d", int64(id)) pub.log.AuditErrf("Failed to submit certificate to CT log: %s", err) return nil, err } chain = append(chain, issuerBundle...) // Add a log URL/pubkey to the cache, if already present the // existing *Log will be returned, otherwise one will be constructed, added // and returned. ctLog, err := pub.ctLogsCache.AddLog(req.LogURL, req.LogPublicKey, pub.userAgent, pub.log) if err != nil { pub.log.AuditErrf("Making Log: %s", err) return nil, err } sct, err := pub.singleLogSubmit(ctx, chain, req.Kind, ctLog) if err != nil { if core.IsCanceled(err) { return nil, err } var body string var rspErr jsonclient.RspError if errors.As(err, &rspErr) && rspErr.StatusCode < 500 { body = string(rspErr.Body) } pub.log.AuditErrf("Failed to submit certificate to CT log at %s: %s Body=%q", ctLog.uri, err, body) return nil, err } sctBytes, err := cttls.Marshal(*sct) if err != nil { return nil, err } return &pubpb.Result{Sct: sctBytes}, nil } func (pub *Impl) singleLogSubmit( ctx context.Context, chain []ct.ASN1Cert, kind pubpb.SubmissionType, ctLog *Log, ) (*ct.SignedCertificateTimestamp, error) { submissionMethod := ctLog.client.AddChain if kind == pubpb.SubmissionType_sct || kind == pubpb.SubmissionType_info { submissionMethod = ctLog.client.AddPreChain } start := time.Now() sct, err := submissionMethod(ctx, chain) took := time.Since(start).Seconds() if err != nil { status := "error" if core.IsCanceled(err) { status = "canceled" } httpStatus := "" var rspError ctClient.RspError if errors.As(err, &rspError) && rspError.StatusCode != 0 { httpStatus = fmt.Sprintf("%d", rspError.StatusCode) } pub.metrics.submissionLatency.With(prometheus.Labels{ "log": ctLog.uri, "type": kind.String(), "status": status, "http_status": httpStatus, }).Observe(took) pub.metrics.errorCount.With(prometheus.Labels{ "log": ctLog.uri, "type": kind.String(), }).Inc() return nil, err } pub.metrics.submissionLatency.With(prometheus.Labels{ "log": ctLog.uri, "type": kind.String(), "status": "success", "http_status": "", }).Observe(took) threshold := uint64(time.Now().Add(time.Minute).UnixMilli()) //nolint: gosec // Current-ish timestamp is guaranteed to fit in a uint64 if sct.Timestamp > threshold { return nil, fmt.Errorf("SCT Timestamp was too far in the future (%d > %d)", sct.Timestamp, threshold) } // For regular certificates, we could get an old SCT, but that shouldn't // happen for precertificates. threshold = uint64(time.Now().Add(-10 * time.Minute).UnixMilli()) //nolint: gosec // Current-ish timestamp is guaranteed to fit in a uint64 if kind != pubpb.SubmissionType_final && sct.Timestamp < threshold { return nil, fmt.Errorf("SCT Timestamp was too far in the past (%d < %d)", sct.Timestamp, threshold) } return sct, nil } // CreateTestingSignedSCT is used by both the publisher tests and ct-test-serv, which is // why it is exported. It creates a signed SCT based on the provided chain. func CreateTestingSignedSCT(req []string, k *ecdsa.PrivateKey, precert bool, timestamp time.Time) []byte { chain := make([]ct.ASN1Cert, len(req)) for i, str := range req { b, err := base64.StdEncoding.DecodeString(str) if err != nil { panic("cannot decode chain") } chain[i] = ct.ASN1Cert{Data: b} } // Generate the internal leaf entry for the SCT etype := ct.X509LogEntryType if precert { etype = ct.PrecertLogEntryType } leaf, err := ct.MerkleTreeLeafFromRawChain(chain, etype, 0) if err != nil { panic(fmt.Sprintf("failed to create leaf: %s", err)) } // Sign the SCT rawKey, _ := x509.MarshalPKIXPublicKey(&k.PublicKey) logID := sha256.Sum256(rawKey) timestampMillis := uint64(timestamp.UnixMilli()) //nolint: gosec // Current-ish timestamp is guaranteed to fit in a uint64 serialized, _ := ct.SerializeSCTSignatureInput(ct.SignedCertificateTimestamp{ SCTVersion: ct.V1, LogID: ct.LogID{KeyID: logID}, Timestamp: timestampMillis, }, ct.LogEntry{Leaf: *leaf}) hashed := sha256.Sum256(serialized) var ecdsaSig struct { R, S *big.Int } ecdsaSig.R, ecdsaSig.S, _ = ecdsa.Sign(rand.Reader, k, hashed[:]) sig, _ := asn1.Marshal(ecdsaSig) // The ct.SignedCertificateTimestamp object doesn't have the needed // `json` tags to properly marshal so we need to transform in into // a struct that does before we can send it off var jsonSCTObj struct { SCTVersion ct.Version `json:"sct_version"` ID string `json:"id"` Timestamp uint64 `json:"timestamp"` Extensions string `json:"extensions"` Signature string `json:"signature"` } jsonSCTObj.SCTVersion = ct.V1 jsonSCTObj.ID = base64.StdEncoding.EncodeToString(logID[:]) jsonSCTObj.Timestamp = timestampMillis ds := ct.DigitallySigned{ Algorithm: cttls.SignatureAndHashAlgorithm{ Hash: cttls.SHA256, Signature: cttls.ECDSA, }, Signature: sig, } jsonSCTObj.Signature, _ = ds.Base64String() jsonSCT, _ := json.Marshal(jsonSCTObj) return jsonSCT } // GetCTBundleForChain takes a slice of *issuance.Certificate(s) // representing a certificate chain and returns a slice of // ct.ASN1Cert(s) in the same order func GetCTBundleForChain(chain []*issuance.Certificate) []ct.ASN1Cert { var ctBundle []ct.ASN1Cert for _, cert := range chain { ctBundle = append(ctBundle, ct.ASN1Cert{Data: cert.Raw}) } return ctBundle }