Revert "Revert "Copy all statsd stats to Prometheus. (#2474)" (#2541)"

This reverts commit 9d9e4941a5 and
restores the statsd prometheus code.

docs/prometheus-stats.md

@@ -0,0 +1,21 @@
+Boulder currently exports stats with both Prometheus and statsd. It is in the
+middle of a transition towards Prometheus, with statsd disappearing soon.
+
+Currently, two stat groups in particular are duplicated:
+
+Stats starting with "Gostats_" are duplicated with the default process-level
+stats exported by the Prometheus library.
+
+Stats starting with "gRPCClient_" are duplicated by the stats generated by
+the go-grpc-prometheus package.
+
+When writing dashboards and alerts in the Prometheus world, we should be careful
+to avoid these two categories, as they will disappear eventually. As a general
+rule, if a stat is available with an all-lowercase name, choose that one, as it
+is probably the Prometheus-native version.
+
+In the long run we will want to create most stats using the native Prometheus
+stat interface, since it allows us to use add labels to metrics, which is very
+useful. For instance, currently our DNS stats distinguish types of queries by
+appending the type to the stat name. This would be more natural as a label in
+Prometheus.

grpc/interceptors.go

@@ -22,6 +22,7 @@ type serverInterceptor struct {
 
 func cleanMethod(m string, trimService bool) string {
 	m = strings.TrimLeft(m, "-")
+	m = strings.Replace(m, "/", "_", -1)
 	if trimService {
 		s := strings.Split(m, "-")
 		if len(s) == 1 {

metrics/auto.go

@@ -0,0 +1,89 @@
+package metrics
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// promAdjust adjusts a name for use by Prometheus: It strips off a single label
+// of prefix (which is always the name of the service, and therefore duplicated
+// by Prometheus' instance labels), and replaces "-" and "." with "_".
+func promAdjust(name string) string {
+	name = strings.Replace(name, "-", "_", -1)
+	labels := strings.Split(name, ".")
+	if len(labels) < 2 {
+		return labels[0]
+	}
+	return strings.Join(labels[1:], "_")
+}
+
+// autoProm implements a bridge from statsd-style metrics to Prometheus-style
+// metrics, automatically registering metrics the first time they are used and
+// memoizing them (since Prometheus doesn't allow repeat registration of the
+// same metric). It is safe for concurrent access.
+type autoProm struct {
+	sync.RWMutex
+	metrics map[string]prometheus.Collector
+}
+
+type maker func(string) prometheus.Collector
+
+func (ap *autoProm) get(name string, make maker) prometheus.Collector {
+	name = promAdjust(name)
+	ap.RLock()
+	result := ap.metrics[name]
+	ap.RUnlock()
+	if result != nil {
+		return result
+	}
+	ap.Lock()
+	defer ap.Unlock()
+
+	// Check once more, since it could have been added while we were locked.
+	if ap.metrics[name] != nil {
+		return ap.metrics[name]
+	}
+	result = make(name)
+	prometheus.MustRegister(result)
+	ap.metrics[name] = result
+	return result
+}
+
+func newAutoProm() *autoProm {
+	return &autoProm{
+		metrics: make(map[string]prometheus.Collector),
+	}
+}
+
+var gauges = newAutoProm()
+var counters = newAutoProm()
+var summaries = newAutoProm()
+
+func autoGauge(name string) prometheus.Gauge {
+	return gauges.get(name, func(cleaned string) prometheus.Collector {
+		return prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: cleaned,
+			Help: "auto",
+		})
+	}).(prometheus.Gauge)
+}
+
+func autoCounter(name string) prometheus.Counter {
+	return counters.get(name, func(cleaned string) prometheus.Collector {
+		return prometheus.NewCounter(prometheus.CounterOpts{
+			Name: cleaned,
+			Help: "auto",
+		})
+	}).(prometheus.Counter)
+}
+
+func autoSummary(name string) prometheus.Summary {
+	return summaries.get(name, func(cleaned string) prometheus.Collector {
+		return prometheus.NewSummary(prometheus.SummaryOpts{
+			Name: cleaned,
+			Help: "auto",
+		})
+	}).(prometheus.Summary)
+}

metrics/auto_test.go

@@ -0,0 +1,50 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestPromAdjust(t *testing.T) {
+	testCases := []struct {
+		input, output string
+	}{
+		{"RA.Foo.Bar", "Foo_Bar"},
+		{"", ""},
+		{"RA-FOO-BAR", "RA_FOO_BAR"},
+		{"RA.FOO-BAR", "FOO_BAR"},
+		{"RA.FOO-BAR", "FOO_BAR"},
+		{"RA", "RA"},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			if promAdjust(tc.input) != tc.output {
+				t.Errorf("expected %q, got %q", tc.input, tc.output)
+			}
+		})
+	}
+}
+
+func TestAutoProm(t *testing.T) {
+	var calledWithName string
+	var madeGauge prometheus.Gauge
+	recorder := func(s string) prometheus.Collector {
+		calledWithName = s
+		madeGauge = prometheus.NewGauge(prometheus.GaugeOpts{Name: "hi", Help: "hi"})
+		return madeGauge
+	}
+	ap := newAutoProm()
+	result := ap.get("foo.bar", recorder)
+	if calledWithName != "bar" {
+		t.Errorf("expected maker function to be called with bar, got %q", calledWithName)
+	}
+	if result != madeGauge {
+		t.Errorf("got back a different gauge than we made")
+	}
+	// Try again, make sure it was memoized again.
+	result2 := ap.get("foo.bar", recorder)
+	if result != result2 {
+		t.Errorf("expected to get same result twice, got a new result")
+	}
+}

metrics/scope.go

@@ -65,6 +65,7 @@ func (s *StatsdScope) Scope() string {
 
 // Inc increments the given stat and adds the Scope's prefix to the name
 func (s *StatsdScope) Inc(stat string, value int64) error {
+	autoCounter(s.prefix + stat).Add(float64(1))
 	return s.statter.Inc(s.prefix+stat, value, 1.0)
 }
 
@@ -75,22 +76,26 @@ func (s *StatsdScope) Dec(stat string, value int64) error {
 
 // Gauge sends a gauge stat and adds the Scope's prefix to the name
 func (s *StatsdScope) Gauge(stat string, value int64) error {
+	autoGauge(s.prefix + stat).Set(float64(value))
 	return s.statter.Gauge(s.prefix+stat, value, 1.0)
 }
 
 // GaugeDelta sends the change in a gauge stat and adds the Scope's prefix to the name
 func (s *StatsdScope) GaugeDelta(stat string, value int64) error {
+	autoGauge(s.prefix + stat).Add(float64(value))
 	return s.statter.GaugeDelta(s.prefix+stat, value, 1.0)
 }
 
 // Timing sends a latency stat and adds the Scope's prefix to the name
 func (s *StatsdScope) Timing(stat string, delta int64) error {
+	autoSummary(s.prefix + stat + "_seconds").Observe(float64(delta))
 	return s.statter.Timing(s.prefix+stat, delta, 1.0)
 }
 
 // TimingDuration sends a latency stat as a time.Duration and adds the Scope's
 // prefix to the name
 func (s *StatsdScope) TimingDuration(stat string, delta time.Duration) error {
+	autoSummary(s.prefix + stat + "_seconds").Observe(delta.Seconds())
 	return s.statter.TimingDuration(s.prefix+stat, delta, 1.0)
 }
 
@@ -101,6 +106,7 @@ func (s *StatsdScope) Set(stat string, value string) error {
 
 // SetInt sets a stat's integer value and adds the Scope's prefix to the name
 func (s *StatsdScope) SetInt(stat string, value int64) error {
+	autoGauge(s.prefix + stat).Set(float64(value))
 	return s.statter.SetInt(s.prefix+stat, value, 1.0)
 }
 

publisher/publisher.go

@@ -115,10 +115,12 @@ func NewLog(uri, b64PK string, logger blog.Logger) (*Log, error) {
 	sanitizedPath := strings.TrimPrefix(url.Path, "/")
 	sanitizedPath = strings.Replace(sanitizedPath, "/", ".", -1)
 
+	sanitizedHost := strings.Replace(url.Host, ":", "_", -1)
+
 	return &Log{
 		logID:    b64PK,
 		uri:      uri,
-		statName: fmt.Sprintf("%s.%s", url.Host, sanitizedPath),
+		statName: fmt.Sprintf("%s.%s", sanitizedHost, sanitizedPath),
 		client:   client,
 		verifier: verifier,
 	}, nil

ra/ra.go

@@ -102,10 +102,10 @@ func NewRegistrationAuthorityImpl(
 		maxNames:                     maxNames,
 		forceCNFromSAN:               forceCNFromSAN,
 		reuseValidAuthz:              reuseValidAuthz,
-		regByIPStats:                 stats.NewScope("RA", "RateLimit", "RegistrationsByIP"),
-		pendAuthByRegIDStats:         stats.NewScope("RA", "RateLimit", "PendingAuthorizationsByRegID"),
-		certsForDomainStats:          stats.NewScope("RA", "RateLimit", "CertificatesForDomain"),
-		totalCertsStats:              stats.NewScope("RA", "RateLimit", "TotalCertificates"),
+		regByIPStats:                 stats.NewScope("RateLimit", "RegistrationsByIP"),
+		pendAuthByRegIDStats:         stats.NewScope("RateLimit", "PendingAuthorizationsByRegID"),
+		certsForDomainStats:          stats.NewScope("RateLimit", "CertificatesForDomain"),
+		totalCertsStats:              stats.NewScope("RateLimit", "TotalCertificates"),
 		publisher:                    pubc,
 	}
 	return ra

test/config-next/ocsp-updater.json

@@ -55,7 +55,7 @@
     "ct": {
       "logs": [
         {
-          "uri": "http://127.0.0.1:4500",
+          "uri": "http://boulder:4500",
           "key": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYggOxPnPkzKBIhTacSYoIfnSL2jPugcbUKx83vFMvk5gKAz/AGe87w20riuPwEGn229hKVbEKHFB61NIqNHC3Q=="
         }
       ]

test/config-next/publisher.json

@@ -41,7 +41,7 @@
     "ct": {
       "logs": [
         {
-          "uri": "http://127.0.0.1:4500",
+          "uri": "http://boulder:4500",
           "key": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYggOxPnPkzKBIhTacSYoIfnSL2jPugcbUKx83vFMvk5gKAz/AGe87w20riuPwEGn229hKVbEKHFB61NIqNHC3Q=="
         }
       ],

test/config/ocsp-updater.json

@@ -51,7 +51,7 @@
     "ct": {
       "logs": [
         {
-          "uri": "http://127.0.0.1:4500",
+          "uri": "http://boulder:4500",
           "key": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYggOxPnPkzKBIhTacSYoIfnSL2jPugcbUKx83vFMvk5gKAz/AGe87w20riuPwEGn229hKVbEKHFB61NIqNHC3Q=="
         }
       ]

test/config/publisher.json

@@ -37,7 +37,7 @@
     "ct": {
       "logs": [
         {
-          "uri": "http://127.0.0.1:4500",
+          "uri": "http://boulder:4500",
           "key": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYggOxPnPkzKBIhTacSYoIfnSL2jPugcbUKx83vFMvk5gKAz/AGe87w20riuPwEGn229hKVbEKHFB61NIqNHC3Q=="
         }
       ],