hook in everywhere, add Noop client in tests

ca/certificate-authority_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	apisign "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/api/sign"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/auth"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
@@ -219,8 +220,9 @@ var NO_NAME_CSR_HEX = "308202523082013a020100300d310b300906035504061302555330820
 	"1778a629b584f2bc059489a236131de9b444adca90218c31a499a485"
 
 func TestIssueCertificate(t *testing.T) {
+	stats, _ := statsd.NewNoopClient(nil)
 	// Audit logger
-	audit, _ := blog.Dial("", "", "tag")
+	audit, _ := blog.Dial("", "", "tag", stats)
 
 	// Decode pre-generated values
 	caKeyPEM, _ := pem.Decode([]byte(CA_KEY_PEM))

cmd/activity-monitor/main.go

@@ -135,14 +135,14 @@ func main() {
 	app := cmd.NewAppShell("activity-monitor")
 
 	app.Action = func(c cmd.Config) {
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
-
-		cmd.FailOnError(err, "Could not connect to Syslog")
-
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
 
 		cmd.FailOnError(err, "Couldn't connect to statsd")
 
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
+
+		cmd.FailOnError(err, "Could not connect to Syslog")
+
 		ch := cmd.AmqpChannel(c.AMQP.Server)
 
 		go cmd.ProfileCmd("AM", stats, auditlogger)

cmd/boulder-ca/main.go

@@ -18,16 +18,16 @@ import (
 func main() {
 	app := cmd.NewAppShell("boulder-ca")
 	app.Action = func(c cmd.Config) {
+		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
+		cmd.FailOnError(err, "Couldn't connect to statsd")
+
 		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
 		cmd.FailOnError(err, "Could not connect to Syslog")
 
 		cai, err := ca.NewCertificateAuthorityImpl(auditlogger, c.CA.Server, c.CA.AuthKey, c.CA.Profile)
 		cmd.FailOnError(err, "Failed to create CA impl")
 
-		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
-		cmd.FailOnError(err, "Couldn't connect to statsd")
-
 		go cmd.ProfileCmd("CA", stats, auditlogger)
 
 

cmd/boulder-ra/main.go

@@ -18,15 +18,15 @@ import (
 func main() {
 	app := cmd.NewAppShell("boulder-ra")
 	app.Action = func(c cmd.Config) {
+		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
+		cmd.FailOnError(err, "Couldn't connect to statsd")
+
 		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
 		cmd.FailOnError(err, "Could not connect to Syslog")
 
 		rai := ra.NewRegistrationAuthorityImpl(auditlogger)
 
-		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
-		cmd.FailOnError(err, "Couldn't connect to statsd")
-
 		go cmd.ProfileCmd("RA", stats, auditlogger)
 
 

cmd/boulder-sa/main.go

@@ -21,16 +21,16 @@ import (
 func main() {
 	app := cmd.NewAppShell("boulder-sa")
 	app.Action = func(c cmd.Config) {
+		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
+		cmd.FailOnError(err, "Couldn't connect to statsd")
+
 		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
 		cmd.FailOnError(err, "Could not connect to Syslog")
 
 		sai, err := sa.NewSQLStorageAuthority(auditlogger, c.SA.DBDriver, c.SA.DBName)
 		cmd.FailOnError(err, "Failed to create SA impl")
 		
-		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
-		cmd.FailOnError(err, "Couldn't connect to statsd")
-
 		go cmd.ProfileCmd("SA", stats, auditlogger)
 
 

cmd/boulder-va/main.go

@@ -18,13 +18,13 @@ import (
 func main() {
 	app := cmd.NewAppShell("boulder-va")
 	app.Action = func(c cmd.Config) {
-		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
-		cmd.FailOnError(err, "Could not connect to Syslog")
-
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
 		cmd.FailOnError(err, "Couldn't connect to statsd")
 
+		// Set up logging
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
+		cmd.FailOnError(err, "Could not connect to Syslog")
+
 		go cmd.ProfileCmd("VA", stats, auditlogger)
 
 		vai := va.NewValidationAuthorityImpl(auditlogger, c.CA.TestMode)

cmd/boulder-wfe/main.go

@@ -68,12 +68,12 @@ func main() {
 	app := cmd.NewAppShell("boulder-wfe")
 	app.Action = func(c cmd.Config) {
 		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
-		cmd.FailOnError(err, "Could not connect to Syslog")
-
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
 		cmd.FailOnError(err, "Couldn't connect to statsd")
 
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
+		cmd.FailOnError(err, "Could not connect to Syslog")
+
 		wfe := wfe.NewWebFrontEndImpl(auditlogger)
 		rac, sac, closeChan := setupWFE(c, auditlogger)
 		wfe.RA = &rac

cmd/boulder/main.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	// Load both drivers to allow configuring either
 	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
 	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/mattn/go-sqlite3"
@@ -25,8 +26,11 @@ import (
 func main() {
 	app := cmd.NewAppShell("boulder")
 	app.Action = func(c cmd.Config) {
+		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
+		cmd.FailOnError(err, "Couldn't connect to statsd")
+
 		// Set up logging
-		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag)
+		auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats)
 		cmd.FailOnError(err, "Could not connect to Syslog")
 
 		// Create the components

cmd/shell.go

@@ -166,24 +166,33 @@ func RunUntilSignaled(logger *blog.AuditLogger, server *rpc.AmqpRPCServer, close
 
 func ProfileCmd(profileName string, stats statsd.Statter, logger *blog.AuditLogger) {
 	for {
-		memoryStats := runtime.MemStats{}
+		var memoryStats runtime.MemStats
 		runtime.ReadMemStats(&memoryStats)
 
 		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Goroutines", profileName), int64(runtime.NumGoroutine()), 1.0); err != nil {
-			logger.Warning(fmt.Sprintf("Couldn't send stats to Statsd server: %s", err))
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
 		}
 
-		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.MemAllocd", profileName), int64(memoryStats.Alloc), 1.0); err != nil {
+		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Objects", profileName), int64(memoryStats.HeapObjects), 1.0); err != nil {
-			logger.Warning(fmt.Sprintf("Couldn't send stats to Statsd server: %s", err))
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
 		} 
-		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.MemLookups", profileName), int64(memoryStats.Lookups), 1.0); err != nil {
+		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Idle", profileName), int64(memoryStats.HeapIdle), 1.0); err != nil {
-			logger.Warning(fmt.Sprintf("Couldn't send stats to Statsd server: %s", err))
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
 		}
-		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.MemMallocs", profileName), int64(memoryStats.Mallocs), 1.0); err != nil {
+		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.InUse", profileName), int64(memoryStats.HeapInuse), 1.0); err != nil {
-			logger.Warning(fmt.Sprintf("Couldn't send stats to Statsd server: %s", err))
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
 		}
-		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.MemFrees", profileName), int64(memoryStats.Frees), 1.0); err != nil {
+		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Released", profileName), int64(memoryStats.HeapReleased), 1.0); err != nil {
-			logger.Warning(fmt.Sprintf("Couldn't send stats to Statsd server: %s", err))
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
+		}
+
+		gcPauseAvg := int64(memoryStats.PauseTotalNs) / int64(len(memoryStats.PauseNs))
+
+		if err := stats.Timing(fmt.Sprintf("Gostats.%s.Gc.PauseAvg", profileName), gcPauseAvg, 1.0); err != nil {
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
+		}
+		if err := stats.Gauge(fmt.Sprintf("Gostats.%s.Gc.NextAt", profileName), int64(memoryStats.NextGC), 1.0); err != nil {
+			logger.Warning(fmt.Sprintf("Could not send stats to Statsd server: %s", err))
 		}
 
 		time.Sleep(time.Second)

log/audit-logger.go

@@ -9,6 +9,8 @@ import (
 	"errors"
 	"fmt"
 	"log/syslog"
+
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 )
 
 // The constant used to identify audit-specific messages
@@ -21,43 +23,53 @@ const auditTag = "[AUDIT]"
 // to send a message as an audit event.
 type AuditLogger struct {
 	*syslog.Writer
+	stats statsd.Statter
 }
 
 // Dial establishes a connection to the log daemon by passing through
 // the parameters to the syslog.Dial method.
 // See http://golang.org/pkg/log/syslog/#Dial
-func Dial(network, raddr string, tag string) (*AuditLogger, error) {
+func Dial(network, raddr string, tag string, stats statsd.Statter) (*AuditLogger, error) {
 	syslogger, err := syslog.Dial(network, raddr, syslog.LOG_INFO|syslog.LOG_LOCAL0, tag)
 	if err != nil {
 		return nil, err
 	}
-	return NewAuditLogger(syslogger)
+	return NewAuditLogger(syslogger, stats)
 }
 
 // NewAuditLogger constructs an Audit Logger that decorates a normal
 // System Logger. All methods in log/syslog continue to work.
-func NewAuditLogger(log *syslog.Writer) (*AuditLogger, error) {
+func NewAuditLogger(log *syslog.Writer, stats statsd.Statter) (*AuditLogger, error) {
 	if log == nil {
 		return nil, errors.New("Attempted to use a nil System Logger.")
 	}
-	return &AuditLogger{log}, nil
+	return &AuditLogger{log, stats}, nil
 }
 
 // Audit sends a NOTICE-severity message that is prefixed with the
 // audit tag, for special handling at the upstream system logger.
 func (log *AuditLogger) Audit(msg string) (err error) {
 	err = log.Notice(fmt.Sprintf("%s %s", auditTag, msg))
+
+	log.stats.Inc("Logging.Audit", 1, 1.0)
+
 	return
 }
 
 // Audit can format an error for auditing; it does so at ERR level.
 func (log *AuditLogger) AuditErr(msg error) (err error) {
 	err = log.Err(fmt.Sprintf("%s %s", auditTag, msg))
+
+	log.stats.Inc("Logging.Error", 1, 1.0)
+
 	return
 }
 
 // Warning formats an error for the Warn level.
 func (log *AuditLogger) WarningErr(msg error) (err error) {
 	err = log.Warning(fmt.Sprintf("%s", msg))
+
+	log.stats.Inc("Logging.Warning", 1, 1.0)
+
 	return
 }

policy/policy-authority_test.go

@@ -8,6 +8,7 @@ package policy
 import (
 	"testing"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	"github.com/letsencrypt/boulder/core"
 	blog "github.com/letsencrypt/boulder/log"
 )
@@ -88,8 +89,9 @@ func TestWillingToIssue(t *testing.T) {
 		"www.zombo-.com",
 	}
 
+	stats, _ := statsd.NewNoopClient(nil)
 	// Audit logger
-	audit, _ := blog.Dial("", "", "tag")
+	audit, _ := blog.Dial("", "", "tag", stats)
 
 	pa := NewPolicyAuthorityImpl(audit)
 
@@ -134,8 +136,9 @@ func TestWillingToIssue(t *testing.T) {
 }
 
 func TestChallengesFor(t *testing.T) {
+	stats, _ := statsd.NewNoopClient(nil)
 	// Audit logger
-	audit, _ := blog.Dial("", "", "tag")
+	audit, _ := blog.Dial("", "", "tag", stats)
 
 	pa := NewPolicyAuthorityImpl(audit)
 

ra/registration-authority_test.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"testing"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/signer/local"
 	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/mattn/go-sqlite3"
 	"github.com/letsencrypt/boulder/ca"
@@ -69,8 +70,9 @@ var (
 )
 
 func initAuthorities(t *testing.T) (core.CertificateAuthority, *DummyValidationAuthority, *sa.SQLStorageAuthority, core.RegistrationAuthority) {
+	stats, _ := statsd.NewNoopClient(nil)
 	// Audit logger
-	audit, _ := blog.Dial("", "", "tag")
+	audit, _ := blog.Dial("", "", "tag", stats)
 
 	err := json.Unmarshal(AccountKeyJSON, &AccountKey)
 	test.AssertNotError(t, err, "Failed to unmarshall JWK")