Merge branch 'master' into fix-nonce-race

README.md

@@ -30,6 +30,7 @@ recent version.
 
 Also, Boulder requires Go 1.5. As of September 2015 this version is not yet
 available in OS repositories, so you will have to install from https://golang.org/dl/.
+Add ```${GOPATH}/bin``` to your path.
 
 Ubuntu:
 
@@ -50,17 +51,19 @@ or
 (On OS X, using port, you will have to add `CGO_CFLAGS="-I/opt/local/include" CGO_LDFLAGS="-L/opt/local/lib"` to your environment or `go` invocations.)
    
     > go get bitbucket.org/liamstask/goose/cmd/goose
+    > go get github.com/jsha/listenbuddy
     > go get github.com/letsencrypt/boulder/ # Ignore errors about no buildable files
     > cd $GOPATH/src/github.com/letsencrypt/boulder
     > ./test/create_db.sh
     # This starts each Boulder component with test configs. Ctrl-C kills all.
     > ./start.py
     # Run tests
+    > go get -u github.com/golang/lint/golint
     > ./test.sh
 
-Note: `create_db.sh` it uses the root MariaDB user, so if you
+Note: `create_db.sh` it uses the root MariaDB user with the default
-have disabled that account you may have to adjust the file or
+password, so if you have disabled that account or changed the password
-recreate the commands.
+you may have to adjust the file or recreate the commands.
 
 You can also check out the official client from
 https://github.com/letsencrypt/letsencrypt/ and follow the setup

ca/certificate-authority_test.go

@@ -132,7 +132,7 @@ func setup(t *testing.T) *testCtx {
 
 	paDbMap, err := sa.NewDbMap(vars.DBConnPolicy)
 	test.AssertNotError(t, err, "Could not construct dbMap")
-	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, false)
+	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, false, nil)
 	test.AssertNotError(t, err, "Couldn't create PADB")
 	paDBCleanUp := test.ResetPolicyTestDatabase(t)
 

cmd/boulder-ca/main.go

@@ -27,6 +27,10 @@ func main() {
 		cmd.FailOnError(err, "Could not connect to Syslog")
 		auditlogger.Info(app.VersionString())
 
+		// Validate PA config and set defaults if needed
+		cmd.FailOnError(c.PA.CheckChallenges(), "Invalid PA configuration")
+		c.PA.SetDefaultChallengesIfEmpty()
+
 		// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
 		defer auditlogger.AuditPanic()
 
@@ -36,7 +40,7 @@ func main() {
 
 		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
 		cmd.FailOnError(err, "Couldn't connect to policy database")
-		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist)
+		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges)
 		cmd.FailOnError(err, "Couldn't create PA")
 
 		cai, err := ca.NewCertificateAuthorityImpl(c.CA, clock.Default(), stats, c.Common.IssuerCert)

cmd/boulder-ra/main.go

@@ -31,6 +31,10 @@ func main() {
 		cmd.FailOnError(err, "Could not connect to Syslog")
 		auditlogger.Info(app.VersionString())
 
+		// Validate PA config and set defaults if needed
+		cmd.FailOnError(c.PA.CheckChallenges(), "Invalid PA configuration")
+		c.PA.SetDefaultChallengesIfEmpty()
+
 		// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
 		defer auditlogger.AuditPanic()
 
@@ -40,7 +44,7 @@ func main() {
 
 		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
 		cmd.FailOnError(err, "Couldn't connect to policy database")
-		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist)
+		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges)
 		cmd.FailOnError(err, "Couldn't create PA")
 
 		rateLimitPolicies, err := cmd.LoadRateLimitPolicies(c.RA.RateLimitPoliciesFilename)
@@ -68,7 +72,7 @@ func main() {
 
 		var dc *ra.DomainCheck
 		if c.RA.UseIsSafeDomain {
-			dc = &ra.DomainCheck{&vac}
+			dc = &ra.DomainCheck{VA: &vac}
 		}
 
 		rai := ra.NewRegistrationAuthorityImpl(clock.Default(), auditlogger, stats,

cmd/cert-checker/main.go

@@ -76,8 +76,8 @@ type certChecker struct {
 	issuedReport report
 }
 
-func newChecker(saDbMap *gorp.DbMap, paDbMap *gorp.DbMap, clk clock.Clock, enforceWhitelist bool) certChecker {
+func newChecker(saDbMap *gorp.DbMap, paDbMap *gorp.DbMap, clk clock.Clock, enforceWhitelist bool, challengeTypes map[string]bool) certChecker {
-	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, enforceWhitelist)
+	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, enforceWhitelist, challengeTypes)
 	cmd.FailOnError(err, "Failed to create PA")
 	c := certChecker{
 		pa:    pa,
@@ -235,6 +235,10 @@ func main() {
 	}
 
 	app.Action = func(c cmd.Config) {
+		// Validate PA config and set defaults if needed
+		cmd.FailOnError(c.PA.CheckChallenges(), "Invalid PA configuration")
+		c.PA.SetDefaultChallengesIfEmpty()
+
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
 		cmd.FailOnError(err, "Couldn't connect to statsd")
 
@@ -250,7 +254,7 @@ func main() {
 		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
 		cmd.FailOnError(err, "Could not connect to policy database")
 
-		checker := newChecker(saDbMap, paDbMap, clock.Default(), c.PA.EnforcePolicyWhitelist)
+		checker := newChecker(saDbMap, paDbMap, clock.Default(), c.PA.EnforcePolicyWhitelist, c.PA.Challenges)
 		auditlogger.Info("# Getting certificates issued in the last 90 days")
 
 		// Since we grab certificates in batches we don't want this to block, when it

cmd/cert-checker/main_test.go

@@ -47,7 +47,7 @@ func BenchmarkCheckCert(b *testing.B) {
 		fmt.Printf("Failed to truncate tables: %s\n", err)
 	}()
 
-	checker := newChecker(saDbMap, paDbMap, clock.Default(), false)
+	checker := newChecker(saDbMap, paDbMap, clock.Default(), false, nil)
 	testKey, _ := rsa.GenerateKey(rand.Reader, 1024)
 	expiry := time.Now().AddDate(0, 0, 1)
 	serial := big.NewInt(1337)
@@ -89,7 +89,7 @@ func TestCheckCert(t *testing.T) {
 	fc := clock.NewFake()
 	fc.Add(time.Hour * 24 * 90)
 
-	checker := newChecker(saDbMap, paDbMap, fc, false)
+	checker := newChecker(saDbMap, paDbMap, fc, false, nil)
 
 	issued := checker.clock.Now().Add(-time.Hour * 24 * 45)
 	goodExpiry := issued.Add(checkPeriod)
@@ -181,7 +181,7 @@ func TestGetAndProcessCerts(t *testing.T) {
 	test.AssertNotError(t, err, "Couldn't connect to policy database")
 	fc := clock.NewFake()
 
-	checker := newChecker(saDbMap, paDbMap, fc, false)
+	checker := newChecker(saDbMap, paDbMap, fc, false, nil)
 	sa, err := sa.NewSQLStorageAuthority(saDbMap, fc)
 	test.AssertNotError(t, err, "Couldn't create SA to insert certificates")
 	saCleanUp := test.ResetSATestDatabase(t)

cmd/shell.go

@@ -260,10 +260,37 @@ type CAConfig struct {
 }
 
 // PAConfig specifies how a policy authority should connect to its
-// database, and what policies it should enforce.
+// database, what policies it should enforce, and what challenges
+// it should offer.
 type PAConfig struct {
 	DBConnect              string
 	EnforcePolicyWhitelist bool
+	Challenges             map[string]bool
+}
+
+// CheckChallenges checks whether the list of challenges in the PA config
+// actually contains valid challenge names
+func (pc PAConfig) CheckChallenges() error {
+	for name := range pc.Challenges {
+		if !core.ValidChallenge(name) {
+			return fmt.Errorf("Invalid challenge in PA config: %s", name)
+		}
+	}
+	return nil
+}
+
+// SetDefaultChallengesIfEmpty sets a default list of challenges if no
+// challenges are enabled in the PA config.  The set of challenges specified
+// corresponds to the set that was hard-coded before these configuration
+// options were added.
+func (pc *PAConfig) SetDefaultChallengesIfEmpty() {
+	if len(pc.Challenges) == 0 {
+		pc.Challenges = map[string]bool{}
+		pc.Challenges[core.ChallengeTypeSimpleHTTP] = true
+		pc.Challenges[core.ChallengeTypeDVSNI] = true
+		pc.Challenges[core.ChallengeTypeHTTP01] = true
+		pc.Challenges[core.ChallengeTypeTLSSNI01] = true
+	}
 }
 
 // KeyConfig should contain either a File path to a PEM-format private key,

cmd/shell_test.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/letsencrypt/boulder/test"
+)
+
+var (
+	validPAConfig = []byte(`{
+  "dbConnect": "dummyDBConnect",
+  "enforcePolicyWhitelist": false,
+  "challenges": { "simpleHttp": true }
+}`)
+	invalidPAConfig = []byte(`{
+  "dbConnect": "dummyDBConnect",
+  "enforcePolicyWhitelist": false,
+  "challenges": { "nonsense": true }
+}`)
+	noChallengesPAConfig = []byte(`{
+  "dbConnect": "dummyDBConnect",
+  "enforcePolicyWhitelist": false
+}`)
+)
+
+func TestPAConfigUnmarshal(t *testing.T) {
+	var pc1 PAConfig
+	err := json.Unmarshal(validPAConfig, &pc1)
+	test.AssertNotError(t, err, "Failed to unmarshal PAConfig")
+	test.AssertNotError(t, pc1.CheckChallenges(), "Flagged valid challenges as bad")
+
+	var pc2 PAConfig
+	err = json.Unmarshal(invalidPAConfig, &pc2)
+	test.AssertNotError(t, err, "Failed to unmarshal PAConfig")
+	test.AssertError(t, pc2.CheckChallenges(), "Considered invalid challenges as good")
+
+	var pc3 PAConfig
+	err = json.Unmarshal(noChallengesPAConfig, &pc3)
+	test.AssertNotError(t, err, "Failed to unmarshal PAConfig")
+	test.AssertNotError(t, pc3.CheckChallenges(), "Somehow found a bad challenge among none")
+	pc3.SetDefaultChallengesIfEmpty()
+	test.Assert(t, len(pc3.Challenges) == 4, "Incorrect number of challenges in default set")
+}

core/core_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/go-jose"
+	"github.com/letsencrypt/boulder/test"
 )
 
 // challenges.go
@@ -56,6 +57,15 @@ func TestChallenges(t *testing.T) {
 	if !dns01.IsSane(false) {
 		t.Errorf("New dns-01 challenge is not sane: %v", dns01)
 	}
+
+	// TODO(#894): Remove these lines
+	test.Assert(t, ValidChallenge(ChallengeTypeSimpleHTTP), "Refused valid challenge")
+	test.Assert(t, ValidChallenge(ChallengeTypeDVSNI), "Refused valid challenge")
+
+	test.Assert(t, ValidChallenge(ChallengeTypeHTTP01), "Refused valid challenge")
+	test.Assert(t, ValidChallenge(ChallengeTypeTLSSNI01), "Refused valid challenge")
+	test.Assert(t, ValidChallenge(ChallengeTypeDNS01), "Refused valid challenge")
+	test.Assert(t, !ValidChallenge("nonsense-71"), "Accepted invalid challenge")
 }
 
 // objects.go

core/objects.go

@@ -97,6 +97,27 @@ const (
 	ChallengeTypeDNS01      = "dns-01"
 )
 
+// ValidChallenge tests whether the provided string names a known challenge
+func ValidChallenge(name string) bool {
+	switch name {
+	// TODO(#894): Delete these lines
+	case ChallengeTypeSimpleHTTP:
+		fallthrough
+	case ChallengeTypeDVSNI:
+		fallthrough
+
+	case ChallengeTypeHTTP01:
+		fallthrough
+	case ChallengeTypeTLSSNI01:
+		fallthrough
+	case ChallengeTypeDNS01:
+		return true
+
+	default:
+		return false
+	}
+}
+
 // TLSSNISuffix is appended to pseudo-domain names in DVSNI challenges
 const TLSSNISuffix = "acme.invalid"
 

policy/policy-authority.go

@@ -24,10 +24,11 @@ type PolicyAuthorityImpl struct {
 	DB  *PolicyAuthorityDatabaseImpl
 
 	EnforceWhitelist  bool
+	enabledChallenges map[string]bool
 }
 
 // NewPolicyAuthorityImpl constructs a Policy Authority.
-func NewPolicyAuthorityImpl(dbMap *gorp.DbMap, enforceWhitelist bool) (*PolicyAuthorityImpl, error) {
+func NewPolicyAuthorityImpl(dbMap *gorp.DbMap, enforceWhitelist bool, challengeTypes map[string]bool) (*PolicyAuthorityImpl, error) {
 	logger := blog.GetAuditLogger()
 	logger.Notice("Policy Authority Starting")
 
@@ -36,10 +37,12 @@ func NewPolicyAuthorityImpl(dbMap *gorp.DbMap, enforceWhitelist bool) (*PolicyAu
 	if err != nil {
 		return nil, err
 	}
+
 	pa := PolicyAuthorityImpl{
 		log:               logger,
 		DB:                padb,
 		EnforceWhitelist:  enforceWhitelist,
+		enabledChallenges: challengeTypes,
 	}
 
 	return &pa, nil
@@ -204,13 +207,34 @@ func (pa PolicyAuthorityImpl) WillingToIssue(id core.AcmeIdentifier, regID int64
 //
 // Note: Current implementation is static, but future versions may not be.
 func (pa PolicyAuthorityImpl) ChallengesFor(identifier core.AcmeIdentifier, accountKey *jose.JsonWebKey) (challenges []core.Challenge, combinations [][]int, err error) {
-	// TODO(https://github.com/letsencrypt/boulder/issues/894): Update these lines
+	challenges = []core.Challenge{}
-	challenges = []core.Challenge{
+	combinations = [][]int{}
-		core.SimpleHTTPChallenge(accountKey),
+
-		core.DvsniChallenge(accountKey),
+	// TODO(https://github.com/letsencrypt/boulder/issues/894): Remove this block
-		core.HTTPChallenge01(accountKey),
+	if pa.enabledChallenges[core.ChallengeTypeSimpleHTTP] {
-		core.TLSSNIChallenge01(accountKey),
+		challenges = append(challenges, core.SimpleHTTPChallenge(accountKey))
+	}
+
+	// TODO(https://github.com/letsencrypt/boulder/issues/894): Remove this block
+	if pa.enabledChallenges[core.ChallengeTypeDVSNI] {
+		challenges = append(challenges, core.DvsniChallenge(accountKey))
+	}
+
+	if pa.enabledChallenges[core.ChallengeTypeHTTP01] {
+		challenges = append(challenges, core.HTTPChallenge01(accountKey))
+	}
+
+	if pa.enabledChallenges[core.ChallengeTypeTLSSNI01] {
+		challenges = append(challenges, core.TLSSNIChallenge01(accountKey))
+	}
+
+	if pa.enabledChallenges[core.ChallengeTypeDNS01] {
+		challenges = append(challenges, core.DNSChallenge01(accountKey))
+	}
+
+	combinations = make([][]int, len(challenges))
+	for i := range combinations {
+		combinations[i] = []int{i}
 	}
-	combinations = [][]int{[]int{0}, []int{1}, []int{2}, []int{3}}
 	return
 }

policy/policy-authority_test.go

@@ -21,9 +21,17 @@ import (
 
 var log = mocks.UseMockLog()
 
+var enabledChallenges = map[string]bool{
+	core.ChallengeTypeSimpleHTTP: true,
+	core.ChallengeTypeDVSNI:      true,
+	core.ChallengeTypeHTTP01:     true,
+	core.ChallengeTypeTLSSNI01:   true,
+	core.ChallengeTypeDNS01:      true,
+}
+
 func paImpl(t *testing.T) (*PolicyAuthorityImpl, func()) {
 	dbMap, cleanUp := paDBMap(t)
-	pa, err := NewPolicyAuthorityImpl(dbMap, false)
+	pa, err := NewPolicyAuthorityImpl(dbMap, false, enabledChallenges)
 	if err != nil {
 		cleanUp()
 		t.Fatalf("Couldn't create policy implementation: %s", err)
@@ -207,25 +215,19 @@ func TestChallengesFor(t *testing.T) {
 		t.Errorf("Error generating challenges: %v", err)
 	}
 
-	// TODO(https://github.com/letsencrypt/boulder/issues/894): Update these tests
+	test.Assert(t, len(challenges) == len(enabledChallenges), "Wrong number of challenges returned")
-	if len(challenges) != 4 ||
+	test.Assert(t, len(combinations) == len(enabledChallenges), "Wrong number of combinations returned")
-		challenges[0].Type != core.ChallengeTypeSimpleHTTP ||
+	for i, challenge := range challenges {
-		challenges[1].Type != core.ChallengeTypeDVSNI ||
+		test.Assert(t, enabledChallenges[challenge.Type], "Unsupported challenge returned")
-		challenges[2].Type != core.ChallengeTypeHTTP01 ||
+		test.AssertEquals(t, len(combinations[i]), 1)
-		challenges[3].Type != core.ChallengeTypeTLSSNI01 {
+		test.AssertEquals(t, combinations[i][0], i)
-		t.Error("Incorrect challenges returned")
-	}
-	if len(combinations) != 4 ||
-		combinations[0][0] != 0 || combinations[1][0] != 1 ||
-		combinations[2][0] != 2 || combinations[3][0] != 3 {
-		t.Error("Incorrect combinations returned")
 	}
 }
 
 func TestWillingToIssueWithWhitelist(t *testing.T) {
 	dbMap, cleanUp := paDBMap(t)
 	defer cleanUp()
-	pa, err := NewPolicyAuthorityImpl(dbMap, true)
+	pa, err := NewPolicyAuthorityImpl(dbMap, true, nil)
 	test.AssertNotError(t, err, "Couldn't create policy implementation")
 	googID := core.AcmeIdentifier{
 		Type:  core.IdentifierDNS,

ra/registration-authority_test.go

@@ -56,10 +56,15 @@ func (dva *DummyValidationAuthority) IsSafeDomain(req *core.IsSafeDomainRequest)
 	if dva.IsSafeDomainErr != nil {
 		return nil, dva.IsSafeDomainErr
 	}
-	return &core.IsSafeDomainResponse{!dva.IsNotSafe}, nil
+	return &core.IsSafeDomainResponse{IsSafe: !dva.IsNotSafe}, nil
 }
 
 var (
+	SupportedChallenges = map[string]bool{
+		core.ChallengeTypeHTTP01:   true,
+		core.ChallengeTypeTLSSNI01: true,
+	}
+
 	// These values we simulate from the client
 	AccountKeyJSONA = []byte(`{
 		"kty":"RSA",
@@ -155,10 +160,6 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut
 	err = json.Unmarshal(ShortKeyJSON, &ShortKey)
 	test.AssertNotError(t, err, "Failed to unmarshal JWK")
 
-	simpleHTTP := core.SimpleHTTPChallenge(&AccountKeyA)
-	dvsni := core.DvsniChallenge(&AccountKeyA)
-	AuthzInitial.Challenges = []core.Challenge{simpleHTTP, dvsni}
-
 	fc := clock.NewFake()
 
 	dbMap, err := sa.NewDbMap(vars.DBConnSA)
@@ -198,7 +199,7 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut
 		t.Fatalf("Failed to create dbMap: %s", err)
 	}
 	policyDBCleanUp := test.ResetPolicyTestDatabase(t)
-	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, false)
+	pa, err := policy.NewPolicyAuthorityImpl(paDbMap, false, SupportedChallenges)
 	test.AssertNotError(t, err, "Couldn't create PA")
 	ca := ca.CertificateAuthorityImpl{
 		Signer:         signer,
@@ -242,6 +243,10 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut
 
 	AuthzInitial.RegistrationID = Registration.ID
 
+	challenges, combinations, err := pa.ChallengesFor(AuthzInitial.Identifier, &Registration.Key)
+	AuthzInitial.Challenges = challenges
+	AuthzInitial.Combinations = combinations
+
 	AuthzFinal = AuthzInitial
 	AuthzFinal.Status = "valid"
 	exp := time.Now().Add(365 * 24 * time.Hour)
@@ -403,22 +408,12 @@ func TestNewAuthorization(t *testing.T) {
 	test.Assert(t, authz.Status == core.StatusPending, "Initial authz not pending")
 
 	// TODO Verify that challenges are correct
-	// TODO(https://github.com/letsencrypt/boulder/issues/894): Update these lines
+	test.Assert(t, len(authz.Challenges) == len(SupportedChallenges), "Incorrect number of challenges returned")
-	test.Assert(t, len(authz.Challenges) == 4, "Incorrect number of challenges returned")
+	test.Assert(t, SupportedChallenges[authz.Challenges[0].Type], fmt.Sprintf("Unsupported challenge: %s", authz.Challenges[0].Type))
-	test.Assert(t, authz.Challenges[0].Type == core.ChallengeTypeSimpleHTTP, "Challenge 0 not SimpleHTTP")
+	test.Assert(t, SupportedChallenges[authz.Challenges[1].Type], fmt.Sprintf("Unsupported challenge: %s", authz.Challenges[1].Type))
-	test.Assert(t, authz.Challenges[1].Type == core.ChallengeTypeDVSNI, "Challenge 1 not DVSNI")
-
-	// TODO(https://github.com/letsencrypt/boulder/issues/894): Delete these lines
-	test.Assert(t, authz.Challenges[2].Type == core.ChallengeTypeHTTP01, "Challenge 2 not http-00")
-	test.Assert(t, authz.Challenges[3].Type == core.ChallengeTypeTLSSNI01, "Challenge 3 not tlssni-00")
-
 	test.Assert(t, authz.Challenges[0].IsSane(false), "Challenge 0 is not sane")
 	test.Assert(t, authz.Challenges[1].IsSane(false), "Challenge 1 is not sane")
 
-	// TODO(https://github.com/letsencrypt/boulder/issues/894): Delete these lines
-	test.Assert(t, authz.Challenges[2].IsSane(false), "Challenge 2 is not sane")
-	test.Assert(t, authz.Challenges[3].IsSane(false), "Challenge 3 is not sane")
-
 	t.Log("DONE TestNewAuthorization")
 }
 

ra/safe.go

@@ -25,7 +25,7 @@ func (d *DomainCheck) IsSafe(domain string) (bool, error) {
 		return true, nil
 	}
 
-	resp, err := d.VA.IsSafeDomain(&core.IsSafeDomainRequest{domain})
+	resp, err := d.VA.IsSafeDomain(&core.IsSafeDomainRequest{Domain: domain})
 	if err != nil {
 		return false, err
 	}

rpc/amqp-rpc.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"
 
 	"github.com/letsencrypt/boulder/cmd"
@@ -170,6 +171,8 @@ type AmqpRPCServer struct {
 	currentGoroutines              int64
 	maxConcurrentRPCServerRequests int64
 	tooManyRequestsResponse        []byte
+	stats                          statsd.Statter
+	clk                            clock.Clock
 }
 
 // NewAmqpRPCServer creates a new RPC server for the given queue and will begin
@@ -186,12 +189,19 @@ func NewAmqpRPCServer(serverQueue string, maxConcurrentRPCServerRequests int64,
 		reconnectMax = time.Minute
 	}
 
+	stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
+	if err != nil {
+		return nil, err
+	}
+
 	return &AmqpRPCServer{
 		serverQueue:                    serverQueue,
 		connection:                     newAMQPConnector(serverQueue, reconnectBase, reconnectMax),
 		log:                            log,
 		dispatchTable:                  make(map[string]func([]byte) ([]byte, error)),
 		maxConcurrentRPCServerRequests: maxConcurrentRPCServerRequests,
+		clk:   clock.Default(),
+		stats: stats,
 	}, nil
 }
 
@@ -426,14 +436,19 @@ func (rpc *AmqpRPCServer) Start(c cmd.Config) error {
 		select {
 		case msg, ok := <-rpc.connection.messages():
 			if ok {
+				rpc.stats.TimingDuration(fmt.Sprintf("RPC.MessageLag.%s", rpc.serverQueue), rpc.clk.Now().Sub(msg.Timestamp), 1.0)
 				if rpc.maxConcurrentRPCServerRequests > 0 && atomic.LoadInt64(&rpc.currentGoroutines) >= rpc.maxConcurrentRPCServerRequests {
 					rpc.replyTooManyRequests(msg)
+					rpc.stats.Inc(fmt.Sprintf("RPC.CallsDropped.%s", rpc.serverQueue), 1, 1.0)
 					break // this breaks the select, not the for
 				}
+				rpc.stats.Inc(fmt.Sprintf("RPC.Traffic.Rx.%s", rpc.serverQueue), int64(len(msg.Body)), 1.0)
 				go func() {
 					atomic.AddInt64(&rpc.currentGoroutines, 1)
 					defer atomic.AddInt64(&rpc.currentGoroutines, -1)
+					startedProcessing := rpc.clk.Now()
 					rpc.processMessage(msg)
+					rpc.stats.TimingDuration(fmt.Sprintf("RPC.ServerProcessingLatency.%s", msg.Type), time.Since(startedProcessing), 1.0)
 				}()
 			} else {
 				rpc.mu.RLock()
@@ -623,10 +638,7 @@ func (rpc *AmqpRPCCLient) dispatch(method string, body []byte) (string, chan []b
 
 // DispatchSync sends a body to the destination, and blocks waiting on a response.
 func (rpc *AmqpRPCCLient) DispatchSync(method string, body []byte) (response []byte, err error) {
-	rpc.stats.Inc(fmt.Sprintf("RPC.Rate.%s", method), 1, 1.0)
+	rpc.stats.Inc(fmt.Sprintf("RPC.Traffic.Tx.%s", rpc.serverQueue), int64(len(body)), 1.0)
-	rpc.stats.Inc("RPC.Traffic", int64(len(body)), 1.0)
-	rpc.stats.GaugeDelta("RPC.CallsWaiting", 1, 1.0)
-	defer rpc.stats.GaugeDelta("RPC.CallsWaiting", -1, 1.0)
 	callStarted := time.Now()
 	corrID, responseChan := rpc.dispatch(method, body)
 	select {
@@ -638,16 +650,14 @@ func (rpc *AmqpRPCCLient) DispatchSync(method string, body []byte) (response []b
 		}
 		err = unwrapError(rpcResponse.Error)
 		if err != nil {
-			rpc.stats.Inc(fmt.Sprintf("RPC.Latency.%s.Error", method), 1, 1.0)
+			rpc.stats.Inc(fmt.Sprintf("RPC.ClientCallLatency.%s.Error", method), 1, 1.0)
 			return
 		}
-		rpc.stats.Inc("RPC.Rate.Success", 1, 1.0)
+		rpc.stats.TimingDuration(fmt.Sprintf("RPC.ClientCallLatency.%s.Success", method), time.Since(callStarted), 1.0)
-		rpc.stats.TimingDuration(fmt.Sprintf("RPC.Latency.%s.Success", method), time.Since(callStarted), 1.0)
 		response = rpcResponse.ReturnVal
 		return
 	case <-time.After(rpc.timeout):
-		rpc.stats.TimingDuration(fmt.Sprintf("RPC.Latency.%s.Timeout", method), time.Since(callStarted), 1.0)
+		rpc.stats.TimingDuration(fmt.Sprintf("RPC.ClientCallLatency.%s.Timeout", method), time.Since(callStarted), 1.0)
-		rpc.stats.Inc("RPC.Rate.Timeouts", 1, 1.0)
 		rpc.log.Warning(fmt.Sprintf(" [c!][%s] AMQP-RPC timeout [%s]", rpc.clientQueue, method))
 		rpc.mu.Lock()
 		delete(rpc.pending, corrID)

rpc/connection.go

@@ -129,6 +129,7 @@ func (ac *amqpConnector) publish(queueName, corrId, expiration, replyTo, msgType
 			Expiration:    expiration,
 			ReplyTo:       replyTo,
 			Type:          msgType,
+			Timestamp:     ac.clk.Now(),
 		})
 }
 

rpc/connection_test.go

@@ -31,6 +31,7 @@ func setup(t *testing.T) (*amqpConnector, *MockamqpChannel, func()) {
 		},
 		queueName:        "fooqueue",
 		retryTimeoutBase: time.Second,
+		clk:              clock.NewFake(),
 	}
 	return &ac, mockChannel, func() { mockCtrl.Finish() }
 }
@@ -125,6 +126,7 @@ func TestPublish(t *testing.T) {
 			Expiration:    "3000",
 			ReplyTo:       "replyTo",
 			Type:          "testMsg",
+			Timestamp:     ac.clk.Now(),
 		})
 	ac.publish("fooqueue", "03c52e", "3000", "replyTo", "testMsg", []byte("body"))
 }

test.sh

@@ -127,6 +127,8 @@ function build_letsencrypt() {
   run ./venv/bin/pip install -U pip
   run ./venv/bin/pip install -e acme -e . -e letsencrypt-apache -e letsencrypt-nginx
 
+  source ./venv/bin/activate
+
   cd -
 }
 

test/boulder-config.json

@@ -116,7 +116,14 @@
   },
 
   "pa": {
-    "dbConnect": "mysql+tcp://policy@localhost:3306/boulder_policy_integration"
+    "dbConnect": "mysql+tcp://policy@localhost:3306/boulder_policy_integration",
+    "challenges": {
+      "simpleHttp": true,
+      "dvsni": true,
+      "http-01": true,
+      "tls-sni-01": true,
+      "dns-01": true
+    }
   },
 
   "ra": {

test/integration-test.py

@@ -209,7 +209,7 @@ def run_client_tests():
         "Please set LETSENCRYPT_PATH env variable to point at "
         "initialized (virtualenv) client repo root")
     test_script_path = os.path.join(root, 'tests', 'boulder-integration.sh')
-    cmd = "source %s/venv/bin/activate && SIMPLE_HTTP_PORT=5002 %s" % (root, test_script_path)
+    cmd = "SIMPLE_HTTP_PORT=5002 %s" % (test_script_path)
     if subprocess.Popen(cmd, shell=True, cwd=root, executable='/bin/bash').wait() != 0:
         die(ExitStatus.PythonFailure)
 

va/gsb.go

@@ -29,7 +29,7 @@ func (va *ValidationAuthorityImpl) IsSafeDomain(req *core.IsSafeDomainRequest) (
 	va.stats.Inc("VA.IsSafeDomain.Requests", 1, 1.0)
 	if va.SafeBrowsing == nil {
 		va.stats.Inc("VA.IsSafeDomain.Skips", 1, 1.0)
-		return &core.IsSafeDomainResponse{true}, nil
+		return &core.IsSafeDomainResponse{IsSafe: true}, nil
 	}
 
 	list, err := va.SafeBrowsing.IsListed(req.Domain)
@@ -37,7 +37,7 @@ func (va *ValidationAuthorityImpl) IsSafeDomain(req *core.IsSafeDomainRequest) (
 		va.stats.Inc("VA.IsSafeDomain.Errors", 1, 1.0)
 		if err == safebrowsing.ErrOutOfDateHashes {
 			va.stats.Inc("VA.IsSafeDomain.OutOfDateHashErrors", 1, 1.0)
-			return &core.IsSafeDomainResponse{true}, nil
+			return &core.IsSafeDomainResponse{IsSafe: true}, nil
 		}
 		return nil, err
 	}
@@ -48,5 +48,5 @@ func (va *ValidationAuthorityImpl) IsSafeDomain(req *core.IsSafeDomainRequest) (
 	} else {
 		va.stats.Inc("VA.IsSafeDomain.Status.Bad", 1, 1.0)
 	}
-	return &core.IsSafeDomainResponse{status}, nil
+	return &core.IsSafeDomainResponse{IsSafe: status}, nil
 }

va/gsb_test.go

@@ -34,25 +34,25 @@ func TestIsSafeDomain(t *testing.T) {
 	sbc.EXPECT().IsListed("outofdate.com").Return("", safebrowsing.ErrOutOfDateHashes)
 	va := NewValidationAuthorityImpl(&PortConfig{}, sbc, stats, clock.NewFake())
 
-	resp, err := va.IsSafeDomain(&core.IsSafeDomainRequest{"good.com"})
+	resp, err := va.IsSafeDomain(&core.IsSafeDomainRequest{Domain: "good.com"})
 	if err != nil {
 		t.Errorf("good.com: want no error, got '%s'", err)
 	}
 	if !resp.IsSafe {
 		t.Errorf("good.com: want true, got %t", resp.IsSafe)
 	}
-	resp, err = va.IsSafeDomain(&core.IsSafeDomainRequest{"bad.com"})
+	resp, err = va.IsSafeDomain(&core.IsSafeDomainRequest{Domain: "bad.com"})
 	if err != nil {
 		t.Errorf("bad.com: want no error, got '%s'", err)
 	}
 	if resp.IsSafe {
 		t.Errorf("bad.com: want false, got %t", resp.IsSafe)
 	}
-	_, err = va.IsSafeDomain(&core.IsSafeDomainRequest{"errorful.com"})
+	_, err = va.IsSafeDomain(&core.IsSafeDomainRequest{Domain: "errorful.com"})
 	if err == nil {
 		t.Errorf("errorful.com: want error, got none")
 	}
-	resp, err = va.IsSafeDomain(&core.IsSafeDomainRequest{"outofdate.com"})
+	resp, err = va.IsSafeDomain(&core.IsSafeDomainRequest{Domain: "outofdate.com"})
 	if err != nil {
 		t.Errorf("outofdate.com: want no error, got '%s'", err)
 	}
@@ -67,7 +67,7 @@ func TestAllowNilInIsSafeDomain(t *testing.T) {
 
 	// Be cool with a nil SafeBrowsing. This will happen in prod when we have
 	// flag mismatch between the VA and RA.
-	resp, err := va.IsSafeDomain(&core.IsSafeDomainRequest{"example.com"})
+	resp, err := va.IsSafeDomain(&core.IsSafeDomainRequest{Domain: "example.com"})
 	if err != nil {
 		t.Errorf("nil SafeBrowsing, unexpected error: %s", err)
 	} else if !resp.IsSafe {