Merge pull request #599 from letsencrypt/mysql

use MariaDB in integration tests and start.py

.travis.yml

@@ -17,11 +17,13 @@ addons:
       - libffi-dev
       - ca-certificates
       - rsyslog
+  mariadb: "10.0"
 
 sudo: false
 
 services:
   - rabbitmq
+  - mysql
 
 matrix:
   fast_finish: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.4.2
+FROM golang:1.5rc1
 
 MAINTAINER J.C. Jones "jjones@letsencrypt.org"
 MAINTAINER William Budington "bill@eff.org"
@@ -13,12 +13,13 @@ RUN apt-get update && \
     apt-transport-https && \
   echo deb https://deb.nodesource.com/node_0.12 jessie main > /etc/apt/sources.list.d/nodesource.list && \
   apt-get update && \
-  apt-get install -y --no-install-recommends \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     libltdl-dev \
     rsyslog \
     nodejs \
     lsb-release \
     rabbitmq-server \
+    mariadb-server \
     git-core && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/* \
@@ -42,6 +43,8 @@ RUN ./bootstrap/debian.sh && \
 RUN virtualenv --no-site-packages -p python2 venv && \
   ./venv/bin/pip install -r requirements.txt -e acme -e .[dev,docs,testing] -e letsencrypt-apache -e letsencrypt-nginx
 
+ENV LETSENCRYPT_PATH /letsencrypt
+
 # Copy in the Boulder sources
 COPY . /go/src/github.com/letsencrypt/boulder
 
@@ -55,4 +58,9 @@ RUN go install \
   github.com/letsencrypt/boulder/cmd/boulder-wfe
 
 WORKDIR /go/src/github.com/letsencrypt/boulder
-CMD ["bash", "-c", "rsyslogd && /go/bin/boulder"]
+CMD ["bash", "-c", "service mysql start && \
+service rsyslog start && \
+service rabbitmq-server start && \
+cd /go/src/github.com/letsencrypt/boulder/ && \
+./test/create_db.sh && \
+WFE_LISTEN_ADDR=0.0.0.0:4000 ./start.py"]

README.md

@@ -24,11 +24,8 @@ There are several tags available:
 
 A quick-start method for running a Boulder instance is to use one of the example configurations:
 
-```
-> mkdir .boulder-config
-> cp test/boulder-config.json .boulder-config/config.json
-> docker run --name=boulder --read-only=true --rm=true -v $(pwd)/.boulder-config:/boulder:ro -p 4000:4000 quay.io/letsencrypt/boulder:latest boulder
-```
+    docker run -i --name=boulder --read-only=true --rm=true -p 4000:4000 quay.io/letsencrypt/boulder:latest
+
 
 Alternatively, to run all services locally, using AMQP to pass messages between them, you can use:
 
@@ -119,13 +116,17 @@ The full details of how the various ACME operations happen in Boulder are laid o
 Dependencies
 ------------
 
-All dependencies are vendorized under the Godeps directory,
+All Go dependencies are vendorized under the Godeps directory,
 both to [make dependency management
 easier](https://groups.google.com/forum/m/#!topic/golang-dev/nMWoEAG55v8)
 and to [avoid insecure fallback in go
 get](https://github.com/golang/go/issues/9637).
 
-To update dependencies:
+Local development also requires a RabbitMQ installation and MariaDB
+10 installation. MariaDB should be run on port 3306 for the
+default integration tests.
+
+To update the Go dependencies:
 
 ```
 # Disable insecure fallback by blocking port 80.

cmd/boulder-wfe/main.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"
 
 	"github.com/letsencrypt/boulder/cmd"
@@ -75,6 +76,19 @@ func HandlerTimer(handler http.Handler, stats statsd.Statter) http.Handler {
 
 func main() {
 	app := cmd.NewAppShell("boulder-wfe")
+	addrFlag := cli.StringFlag{
+		Name:   "addr",
+		Value:  "",
+		Usage:  "if set, overrides the listenAddr setting in the WFE config",
+		EnvVar: "WFE_LISTEN_ADDR",
+	}
+	app.App.Flags = append(app.App.Flags, addrFlag)
+	app.Config = func(c *cli.Context, config cmd.Config) cmd.Config {
+		if c.GlobalString("addr") != "" {
+			config.WFE.ListenAddress = c.GlobalString("addr")
+		}
+		return config
+	}
 	app.Action = func(c cmd.Config) {
 		// Set up logging
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
@@ -135,6 +149,7 @@ func main() {
 		auditlogger.Info(app.VersionString())
 
 		// Add HandlerTimer to output resp time + success/failure stats to statsd
+
 		auditlogger.Info(fmt.Sprintf("Server running, listening on %s...\n", c.WFE.ListenAddress))
 		err = http.ListenAndServe(c.WFE.ListenAddress, HandlerTimer(h, stats))
 		cmd.FailOnError(err, "Error starting HTTP server")

docs/database/README.md

@@ -1,5 +1,5 @@
 These `.sql` files define the table layout, indicies, relationships, and users default to Boulder. Implementors should use these as starting points for their own configuration.
 
-## Notes
+The currently supported database is MariaDB 10.
 
-Currently, if you use MySQL / MariaDB with Boulder, you must manually append `?parseTime=true"` onto the end of the `dbConnect` configuration fields for each entry. This is related to [Issue #242](https://github.com/letsencrypt/boulder/issues/242).
+    mysql -u root -e "create database boulder_test; create database boulder_development; grant all privileges on boulder_test.* to 'boulder'@'localhost';"

sa/database.go

@@ -9,6 +9,7 @@ import (
 	"database/sql"
 	"fmt"
 	"net/url"
+	"strings"
 
 	// Load both drivers to allow configuring either
 	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
@@ -35,20 +36,11 @@ func NewDbMap(driver string, dbConnect string) (*gorp.DbMap, error) {
 	logger := blog.GetAuditLogger()
 
 	if driver == "mysql" {
-		// Check the parseTime=true DSN is present
-		dbURI, err := url.Parse(dbConnect)
+		var err error
+		dbConnect, err = recombineURLForDB(dbConnect)
 		if err != nil {
 			return nil, err
 		}
-		dsnVals, err := url.ParseQuery(dbURI.RawQuery)
-		if err != nil {
-			return nil, err
-		}
-		if k := dsnVals.Get("parseTime"); k != "true" {
-			dsnVals.Set("parseTime", "true")
-			dbURI.RawQuery = dsnVals.Encode()
-		}
-		dbConnect = dbURI.String()
 	}
 
 	db, err := sql.Open(driver, dbConnect)
@@ -76,6 +68,53 @@ func NewDbMap(driver string, dbConnect string) (*gorp.DbMap, error) {
 	return dbmap, err
 }
 
+// recombineURLForDB transforms a database URL to a URL-like string
+// that the mysql driver can use. The mysql driver needs the Host data
+// to be wrapped in "tcp()" but url.Parse will escape the parentheses
+// and the mysql driver doesn't understand them. So, we can't have
+// "tcp()" in the configs, but can't leave it out before passing it to
+// the mysql driver. Similarly, the driver needs the password and
+// username unescaped. Compromise by doing the leg work if the config
+// says the database URL's scheme is a fake one called
+// "mysqltcp://". See
+// https://github.com/go-sql-driver/mysql/issues/362 for why we have
+// to futz around and avoid URL.String.
+func recombineURLForDB(dbConnect string) (string, error) {
+	dbConnect = strings.TrimSpace(dbConnect)
+	dbURL, err := url.Parse(dbConnect)
+	if err != nil {
+		return "", err
+	}
+
+	if dbURL.Scheme != "mysql+tcp" {
+		format := "given database connection string was not a mysql+tcp:// URL, was %#v"
+		return "", fmt.Errorf(format, dbURL.Scheme)
+	}
+
+	dsnVals, err := url.ParseQuery(dbURL.RawQuery)
+	if err != nil {
+		return "", err
+	}
+
+	// Check the parseTime=true DSN is present
+	if k := dsnVals.Get("parseTime"); k != "true" {
+		dsnVals.Set("parseTime", "true")
+		dbURL.RawQuery = dsnVals.Encode()
+	}
+	user := dbURL.User.Username()
+	passwd, hasPass := dbURL.User.Password()
+	dbConn := ""
+	if user != "" {
+		dbConn = url.QueryEscape(user)
+	}
+	if hasPass {
+		dbConn += ":" + passwd
+	}
+	dbConn += "@tcp(" + dbURL.Host + ")"
+	// TODO(jmhodges): should be dbURL.EscapedPath() but Travis doesn't have 1.5
+	return dbConn + dbURL.Path + "?" + dsnVals.Encode(), nil
+}
+
 // SetSQLDebug enables/disables GORP SQL-level Debugging
 func SetSQLDebug(dbMap *gorp.DbMap, state bool) {
 	dbMap.TraceOff()

test.sh

@@ -53,7 +53,7 @@ update_status() {
   fi
 }
 
-run() {
+function run() {
   echo "$@"
   "$@" 2>&1
   local status=$?
@@ -70,7 +70,7 @@ run() {
   return ${status}
 }
 
-run_and_comment() {
+function run_and_comment() {
   if [ "x${TRAVIS}" = "x" ] || [ "${TRAVIS_PULL_REQUEST}" == "false" ] || [ ! -f "${GITHUB_SECRET_FILE}" ] ; then
     run "$@"
   else
@@ -208,6 +208,10 @@ if [ "${SKIP_INTEGRATION_TESTS}" = "1" ]; then
   exit ${FAILURE}
 fi
 
+if [ "${TRAVIS}" == "true" ]; then
+  ./test/create_db.sh || die "unable to create the boulder database with test/create_db.sh"
+fi
+
 #
 # Integration tests
 #

test/boulder-config.json

@@ -48,8 +48,8 @@
   "ca": {
     "serialPrefix": 255,
     "profile": "ee",
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:",
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true",
     "debugAddr": "localhost:8001",
     "testMode": true,
     "_comment": "This should only be present in testMode. In prod use an HSM.",
@@ -115,8 +115,8 @@
   },
 
   "sa": {
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:",
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true",
     "debugAddr": "localhost:8003"
   },
 
@@ -131,21 +131,21 @@
   },
 
   "revoker": {
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:"
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true"
   },
 
   "ocspResponder": {
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:",
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true",
     "path": "/",
     "listenAddress": "localhost:4001",
     "debugAddr": "localhost:8005"
   },
 
   "ocspUpdater": {
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:",
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true",
     "minTimeToExpiry": "72h",
     "debugAddr": "localhost:8006"
   },
@@ -159,8 +159,8 @@
     "port": "25",
     "username": "cert-master@example.com",
     "password": "password",
-    "dbDriver": "sqlite3",
-    "dbConnect": ":memory:",
+    "dbDriver": "mysql",
+    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_test?parseTime=true",
     "messageLimit": 0,
     "nagTimes": ["24h", "72h", "168h", "336h"],
     "emailTemplate": "test/example-expiration-template",

test/create_db.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+function die() {
+  if [ ! -z "$1" ]; then
+    echo $1 > /dev/stderr
+  fi
+  exit 1
+}
+
+mysql -u root -e "create database boulder_test; grant all privileges on boulder_test.* to 'boulder'@'localhost'" || die "unable to create boulder_test"
+echo "created boulder_test database"