Merge branch 'master' into google-ct

cmd/activity-monitor/main.go

@@ -36,7 +36,8 @@ func main() {
 
 		go cmd.ProfileCmd("AM", stats)
 
-		server.Start(amqpConf)
+		err = server.Start(amqpConf)
+		cmd.FailOnError(err, "Unable to run Activity Monitor")
 	}
 
 	app.Run()

cmd/admin-revoker/main.go

@@ -48,7 +48,9 @@ func setupContext(context *cli.Context) (rpc.RegistrationAuthorityClient, *blog.
 	rac, err := rpc.NewRegistrationAuthorityClient(clientName, amqpConf, stats)
 	cmd.FailOnError(err, "Unable to create CA client")
 
-	dbMap, err := sa.NewDbMap(c.Revoker.DBConnect)
+	dbURL, err := c.Revoker.DBConfig.URL()
+	cmd.FailOnError(err, "Couldn't load DB URL")
+	dbMap, err := sa.NewDbMap(dbURL)
 	cmd.FailOnError(err, "Couldn't setup database connection")
 
 	sac, err := rpc.NewStorageAuthorityClient(clientName, amqpConf, stats)

cmd/boulder-ca/main.go

@@ -74,7 +74,9 @@ func main() {
 
 		go cmd.DebugServer(c.CA.DebugAddr)
 
-		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
+		dbURL, err := c.PA.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		paDbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Couldn't connect to policy database")
 		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges)
 		cmd.FailOnError(err, "Couldn't create PA")

cmd/boulder-ra/main.go

@@ -31,7 +31,9 @@ func main() {
 
 		go cmd.DebugServer(c.RA.DebugAddr)
 
-		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
+		dbURL, err := c.PA.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		paDbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Couldn't connect to policy database")
 		pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges)
 		cmd.FailOnError(err, "Couldn't create PA")

cmd/boulder-sa/main.go

@@ -20,7 +20,9 @@ func main() {
 		saConf := c.SA
 		go cmd.DebugServer(saConf.DebugAddr)
 
-		dbMap, err := sa.NewDbMap(saConf.DBConnect)
+		dbURL, err := saConf.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		dbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Couldn't connect to SA database")
 
 		sai, err := sa.NewSQLStorageAuthority(dbMap, clock.Default())

cmd/cert-checker/main.go

@@ -239,10 +239,14 @@ func main() {
 		cmd.FailOnError(c.PA.CheckChallenges(), "Invalid PA configuration")
 		c.PA.SetDefaultChallengesIfEmpty()
 
-		saDbMap, err := sa.NewDbMap(c.CertChecker.DBConnect)
+		saDbURL, err := c.CertChecker.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		saDbMap, err := sa.NewDbMap(saDbURL)
 		cmd.FailOnError(err, "Could not connect to database")
 
-		paDbMap, err := sa.NewDbMap(c.PA.DBConnect)
+		paDbURL, err := c.PA.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		paDbMap, err := sa.NewDbMap(paDbURL)
 		cmd.FailOnError(err, "Could not connect to policy database")
 
 		checker := newChecker(saDbMap, paDbMap, clock.Default(), c.PA.EnforcePolicyWhitelist, c.PA.Challenges)

cmd/config.go

@@ -9,6 +9,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"strings"
 	"time"
 
 	cfsslConfig "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
@@ -64,8 +66,7 @@ type Config struct {
 
 	SA struct {
 		ServiceConfig
-
+		DBConfig
-		DBConnect string
 
 		MaxConcurrentRPCServerRequests int64
 	}
@@ -91,7 +92,7 @@ type Config struct {
 	Syslog SyslogConfig
 
 	Revoker struct {
-		DBConnect string
+		DBConfig
 		// The revoker isn't a long running service, so doesn't get a full
 		// ServiceConfig, just an AMQPConfig.
 		AMQP *AMQPConfig
@@ -99,14 +100,13 @@ type Config struct {
 
 	Mailer struct {
 		ServiceConfig
+		DBConfig
 
 		Server   string
 		Port     string
 		Username string
 		Password string
 
-		DBConnect string
-
 		CertLimit int
 		NagTimes  []string
 		// How much earlier (than configured nag intervals) to
@@ -119,10 +119,12 @@ type Config struct {
 
 	OCSPResponder struct {
 		ServiceConfig
+		DBConfig
 
 		// Source indicates the source of pre-signed OCSP responses to be used. It
 		// can be a DBConnect string or a file URL. The file URL style is used
 		// when responding from a static file for intermediates and roots.
+		// If DBConfig has non-empty fields, it takes precedence over this.
 		Source string
 
 		Path          string
@@ -167,9 +169,10 @@ type Config struct {
 	}
 
 	CertChecker struct {
+		DBConfig
+
 		Workers             int
 		ReportDirectoryPath string
-		DBConnect           string
 	}
 
 	SubscriberAgreementURL string
@@ -183,9 +186,32 @@ type ServiceConfig struct {
 	AMQP      *AMQPConfig
 }
 
+// DBConfig defines how to connect to a database. The connect string may be
+// stored in a file separate from the config, because it can contain a password,
+// which we want to keep out of configs.
+type DBConfig struct {
+	DBConnect string
+	// A file containing a connect URL for the DB.
+	DBConnectFile string
+}
+
+// URL returns the DBConnect URL represented by this DBConfig object, either
+// loading it from disk or returning a default value.
+func (d *DBConfig) URL() (string, error) {
+	if d.DBConnectFile != "" {
+		url, err := ioutil.ReadFile(d.DBConnectFile)
+		return string(url), err
+	}
+	return d.DBConnect, nil
+}
+
 // AMQPConfig describes how to connect to AMQP, and how to speak to each of the
 // RPC services we offer via AMQP.
 type AMQPConfig struct {
+	// A file from which the AMQP Server URL will be read. This allows secret
+	// values (like the password) to be stored separately from the main config.
+	ServerURLFile string
+	// AMQP server URL, including username and password.
 	Server    string
 	Insecure  bool
 	RA        *RPCServerConfig
@@ -203,15 +229,28 @@ type AMQPConfig struct {
 	}
 }
 
+// ServerURL returns the appropriate server URL for this object, which may
+// involve reading from a file.
+func (a *AMQPConfig) ServerURL() (string, error) {
+	if a.ServerURLFile != "" {
+		url, err := ioutil.ReadFile(a.ServerURLFile)
+		return strings.TrimRight(string(url), "\n"), err
+	}
+	if a.Server == "" {
+		return "", fmt.Errorf("Missing AMQP server URL")
+	}
+	return a.Server, nil
+}
+
 // CAConfig structs have configuration information for the certificate
 // authority, including database parameters as well as controls for
 // issued certificates.
 type CAConfig struct {
 	ServiceConfig
+	DBConfig
 
 	Profile      string
 	TestMode     bool
-	DBConnect    string
 	SerialPrefix int
 	Key          KeyConfig
 	// LifespanOCSP is how long OCSP responses are valid for; It should be longer
@@ -233,7 +272,7 @@ type CAConfig struct {
 // database, what policies it should enforce, and what challenges
 // it should offer.
 type PAConfig struct {
-	DBConnect              string
+	DBConfig
 	EnforcePolicyWhitelist bool
 	Challenges             map[string]bool
 }
@@ -290,7 +329,7 @@ type RPCServerConfig struct {
 // for the OCSP (and SCT) updater
 type OCSPUpdaterConfig struct {
 	ServiceConfig
-	DBConnect string
+	DBConfig
 
 	NewCertificateWindow     ConfigDuration
 	OldOCSPWindow            ConfigDuration

cmd/expiration-mailer/main.go

@@ -231,10 +231,12 @@ func main() {
 		go cmd.DebugServer(c.Mailer.DebugAddr)
 
 		// Configure DB
-		dbMap, err := sa.NewDbMap(c.Mailer.DBConnect)
+		dbURL, err := c.Mailer.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		dbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Could not connect to database")
 
-		amqpConf := c.SA.AMQP
+		amqpConf := c.Mailer.AMQP
 		sac, err := rpc.NewStorageAuthorityClient(clientName, amqpConf, stats)
 		cmd.FailOnError(err, "Failed to create SA client")
 

cmd/external-cert-importer/main.go

@@ -150,7 +150,9 @@ func main() {
 
 	app.Action = func(c cmd.Config, stats statsd.Statter, auditlogger *blog.AuditLogger) {
 		// Configure DB
-		dbMap, err := sa.NewDbMap(c.PA.DBConnect)
+		dbURL, err := c.PA.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		dbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Could not connect to database")
 
 		dbMap.AddTableWithName(core.ExternalCert{}, "externalCerts").SetKeys(false, "SHA1")

cmd/ocsp-responder/main.go

@@ -141,7 +141,14 @@ func main() {
 
 		config := c.OCSPResponder
 		var source cfocsp.Source
-		url, err := url.Parse(config.Source)
+
+		// DBConfig takes precedence over Source, if present.
+		dbConnect, err := config.DBConfig.URL()
+		cmd.FailOnError(err, "Reading DB config")
+		if dbConnect == "" {
+			dbConnect = config.Source
+		}
+		url, err := url.Parse(dbConnect)
 		cmd.FailOnError(err, fmt.Sprintf("Source was not a URL: %s", config.Source))
 
 		if url.Scheme == "mysql+tcp" {

cmd/ocsp-updater/main.go

@@ -560,7 +560,9 @@ func main() {
 		go cmd.ProfileCmd("OCSP-Updater", stats)
 
 		// Configure DB
-		dbMap, err := sa.NewDbMap(conf.DBConnect)
+		dbURL, err := conf.DBConfig.URL()
+		cmd.FailOnError(err, "Couldn't load DB URL")
+		dbMap, err := sa.NewDbMap(dbURL)
 		cmd.FailOnError(err, "Could not connect to database")
 
 		cac, pubc, sac := setupClients(conf, stats)

cmd/policy-loader/main.go

@@ -110,7 +110,9 @@ func setupFromContext(context *cli.Context) (*policy.PolicyAuthorityDatabaseImpl
 	err = json.Unmarshal(configJSON, &c)
 	cmd.FailOnError(err, "Couldn't unmarshal configuration object")
 
-	dbMap, err := sa.NewDbMap(c.PA.DBConnect)
+	dbURL, err := c.PA.DBConfig.URL()
+	cmd.FailOnError(err, "Couldn't load DB URL")
+	dbMap, err := sa.NewDbMap(dbURL)
 	cmd.FailOnError(err, "Failed to create DB map")
 
 	padb, err := policy.NewPolicyAuthorityDatabaseImpl(dbMap)

rpc/amqp-rpc.go

@@ -297,14 +297,19 @@ func makeAmqpChannel(conf *cmd.AMQPConfig) (*amqp.Channel, error) {
 
 	log := blog.GetAuditLogger()
 
+	serverURL, err := conf.ServerURL()
+	if err != nil {
+		return nil, err
+	}
+
 	if conf.Insecure == true {
 		// If the Insecure flag is true, then just go ahead and connect
-		conn, err = amqp.Dial(conf.Server)
+		conn, err = amqp.Dial(serverURL)
 	} else {
 		// The insecure flag is false or not set, so we need to load up the options
 		log.Info("AMQPS: Loading TLS Options.")
 
-		if strings.HasPrefix(conf.Server, "amqps") == false {
+		if strings.HasPrefix(serverURL, "amqps") == false {
 			err = fmt.Errorf("AMQPS: Not using an AMQPS URL. To use AMQP instead of AMQPS, set insecure=true")
 			return nil, err
 		}
@@ -348,7 +353,7 @@ func makeAmqpChannel(conf *cmd.AMQPConfig) (*amqp.Channel, error) {
 			log.Info("AMQPS: Configured CA certificate for AMQPS.")
 		}
 
-		conn, err = amqp.DialTLS(conf.Server, cfg)
+		conn, err = amqp.DialTLS(serverURL, cfg)
 	}
 
 	if err != nil {

test/boulder-config.json

@@ -96,7 +96,7 @@
     "maxConcurrentRPCServerRequests": 16,
     "hsmFaultTimeout": "300s",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "serviceQueue": "CA.server",
       "SA": {
@@ -127,7 +127,7 @@
     "maxContactsPerRegistration": 100,
     "debugAddr": "localhost:8002",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "serviceQueue": "RA.server",
       "VA": {
@@ -147,11 +147,11 @@
   },
 
   "sa": {
-    "dbConnect": "mysql+tcp://sa@localhost:3306/boulder_sa_integration",
+    "dbConnectFile": "test/secrets/sa_dburl",
     "maxConcurrentRPCServerRequests": 16,
     "debugAddr": "localhost:8003",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "serviceQueue": "SA.server"
     }
@@ -167,7 +167,7 @@
     },
     "maxConcurrentRPCServerRequests": 16,
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "serviceQueue": "VA.server",
       "RA": {
@@ -182,9 +182,9 @@
   },
 
   "revoker": {
-    "dbConnect": "mysql+tcp://revoker@localhost:3306/boulder_sa_integration",
+    "dbConnectFile": "test/secrets/revoker_dburl",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "RA": {
         "server": "RA.server",
@@ -208,7 +208,7 @@
   },
 
   "ocspUpdater": {
-    "dbConnect": "mysql+tcp://ocsp_update@localhost:3306/boulder_sa_integration",
+    "dbConnectFile": "test/secrets/ocsp_updater_dburl",
     "newCertificateWindow": "1s",
     "oldOCSPWindow": "2s",
     "missingSCTWindow": "1m",
@@ -223,7 +223,7 @@
     "signFailureBackoffMax": "30m",
     "debugAddr": "localhost:8006",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "SA": {
         "server": "SA.server",
@@ -244,7 +244,7 @@
     "debugAddr": "localhost:8007",
     "amqp": {
       "serviceQueue": "Monitor",
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true
     }
   },
@@ -254,19 +254,27 @@
     "port": "25",
     "username": "cert-master@example.com",
     "password": "password",
-    "dbConnect": "mysql+tcp://mailer@localhost:3306/boulder_sa_integration",
+    "dbConnectFile": "test/secrets/mailer_dburl",
     "messageLimit": 0,
     "nagTimes": ["24h", "72h", "168h", "336h"],
     "nagCheckInterval": "24h",
     "emailTemplate": "test/example-expiration-template",
-    "debugAddr": "localhost:8008"
+    "debugAddr": "localhost:8008",
+    "amqp": {
+      "serverURLFile": "test/secrets/amqp_url",
+      "insecure": true,
+      "SA": {
+        "server": "SA.server",
+        "rpcTimeout": "15s"
+      }
+    }
   },
 
   "publisher": {
     "maxConcurrentRPCServerRequests": 16,
     "debugAddr": "localhost:8009",
     "amqp": {
-      "server": "amqp://guest:guest@localhost:5673",
+      "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,
       "serviceQueue": "Publisher.server",
       "SA": {
@@ -294,7 +302,7 @@
   },
 
   "certChecker": {
-    "dbConnect": "mysql+tcp://cert_checker@localhost:3306/boulder_sa_integration"
+    "dbConnectFile": "test/secrets/cert_checker_dburl"
   },
 
   "subscriberAgreementURL": "http://127.0.0.1:4001/terms/v1"

test/secrets/amqp_url

@@ -0,0 +1 @@
+amqp://guest:guest@localhost:5673

test/secrets/cert_checker_db_url

@@ -0,0 +1 @@
+mysql+tcp://cert_checker@localhost:3306/boulder_sa_integration

test/secrets/expiration_mailer_dburl

@@ -0,0 +1 @@
+mysql+tcp://mailer@localhost:3306/boulder_sa_integration

test/secrets/ocsp_updater_dburl

@@ -0,0 +1 @@
+mysql+tcp://ocsp_update@localhost:3306/boulder_sa_integration

test/secrets/pa_dburl

@@ -0,0 +1 @@
+mysql+tcp://policy@localhost:3306/boulder_policy_integration

test/secrets/revoker_dburl

@@ -0,0 +1 @@
+mysql+tcp://revoker@localhost:3306/boulder_sa_integration

test/secrets/sa_dburl

@@ -0,0 +1 @@
+mysql+tcp://sa@localhost:3306/boulder_sa_integration