Run Travis tests in Docker (#1830)

* MariaDB 10.1

* MariaDB 10.1 in Docker

* Run docker stuff.

* Improve test.js error.

* Lower log level

* Revert dockerfile to master

* Export debug ports, set FAKE_DNS, and remove container_name.

* Remove typo.

* Make integration-test.py wait for debug ports.

* Use 10.1 and export more Boulder ports.

* Test updates for Docker

Listen on 0.0.0.0 for utility servers.
Make integration-test.py just wait for ports rather than calling startservers.
Run docker-compose in test.sh.
Remove bypass when database exists.
Separate mailer test into its own function in integration test.
Print better errors in test.js.

* Always bring up mysql container.

* Wait for MySQL to come up.

* Put it in travis-before-install.

* Use 127

* Remove manual docker-up.

* Add ifconfig

* Switch to docker-compose run

* It works!

* Remove some spurious env vars.

* Add bash

* try running it

* Add all deps.

* Pass through env.

* Install everything in the Dockerfile.

* Fix install of ruby

* More improvements

* Revert integration test to run directly
Also remove .git from dockerignore and add some packages.

* Revert integration-test.py to master.

* Stop ignoring test/js

* Start from boulder-tools.

* Add boulder-tools.

* Tweak travis.yml

* Separate out docker-compose pull as install.

* Build in install phase; don't bother with go install in Dockerfile

* Add virtualenv

* Actually build rabbitmq-setup

* Remove FAKE_DNS

* Trivial change

* Pull boulder-tools as a separate step so it gets its own timing info.

* Install certbot and protobuf from repos.

* Use cerbot from debian backports.

* Fix clone

* Remove CERTBOT_PATH

* Updates

* Go back to letsencrypt for build.sh

* Remove certbot volume.

* go back to preinstalled letsencrypt

* Restore ENV

* Remove BASH_ENV

* Adapt reloader test so it psses when run as root.

* Fixups for review.

* Revert test.js

* Revert startservers.py

* Revert Makefile.

.dockerignore

@@ -1,4 +1,2 @@
 bin
 tags
-.git
-test/js

.travis.yml

@@ -9,23 +9,11 @@ addons:
     - boulder
     - boulder-mysql
     - boulder-rabbitmq
-  apt:
-    packages:
-      - lsb-release
-      - python-dev
-      - python-virtualenv
-      - gcc
-      - libaugeas0
-      - libssl-dev
-      - libffi-dev
-      - ca-certificates
-      - rsyslog
-  mariadb: "10.0"
 
-sudo: false
+sudo: required
 
 services:
-  - rabbitmq
+  - docker
 
 matrix:
   fast_finish: true
@@ -43,13 +31,6 @@ branches:
     - release
     - /^test-.*$/
 
-# By providing our own install command we avoid Travis' default Go install
-# command, which runs `go get`. We specifically want to avoid that because we
-# want to ensure all our dependencies are vendored.
-install:
-  - travis_retry test/travis-before-install.sh
-  - cd $GOPATH/src/github.com/letsencrypt/boulder
-
 env:
   global:
     - PATH=$HOME/bin:$PATH # protoc gets installed here
@@ -61,6 +42,10 @@ env:
     - RUN="integration" BOULDER_CONFIG="test/boulder-config-next.json"
     - RUN="unit"
 
-script:
-  - bash test.sh
+install:
+  - docker-compose pull
+  - docker pull letsencrypt/boulder-tools
+  - docker-compose build
 
+script:
+  - docker-compose run -e RUN="${RUN}" -e TRAVIS="${TRAVIS}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" boulder  ./test.sh

Dockerfile

@@ -1,34 +1,24 @@
-FROM golang:1.5
-
-MAINTAINER J.C. Jones "jjones@letsencrypt.org"
-MAINTAINER William Budington "bill@eff.org"
-
-# Install dependencies packages
-RUN apt-get update && apt-get install -y \
-	libltdl-dev \
-	mariadb-client-core-10.0 \
-	nodejs \
-	rsyslog \
-	softhsm \
-	--no-install-recommends \
-	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Install port forwarder, database migration tool and go lint
-RUN go get -v \
-	github.com/jsha/listenbuddy \
-	bitbucket.org/liamstask/goose/cmd/goose \
-	github.com/golang/lint/golint
+FROM letsencrypt/boulder-tools:latest
 
 # Boulder exposes its web application at port TCP 4000
 EXPOSE 4000 4002 4003 8053 8055
 
 ENV GO15VENDOREXPERIMENT 1
+ENV GOBIN /go/src/github.com/letsencrypt/boulder/bin
+ENV PATH /go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
+ENV GOPATH /go
+
+RUN adduser --disabled-password --gecos "" --home /go/src/github.com/letsencrypt/boulder -q buser
+RUN chown -R buser /go/
 
 WORKDIR /go/src/github.com/letsencrypt/boulder
 
-ENTRYPOINT [ "./test/entrypoint.sh" ]
-
 # Copy in the Boulder sources
-COPY . /go/src/github.com/letsencrypt/boulder
+COPY . .
+RUN mkdir bin
+RUN go install ./cmd/rabbitmq-setup
+COPY ./test/certbot /go/bin/
 
-RUN GOBIN=/go/src/github.com/letsencrypt/boulder/bin go install  ./...
+RUN chown -R buser /go/
+
+ENTRYPOINT [ "./test/entrypoint.sh" ]

README.md

@@ -27,7 +27,7 @@ setting](https://groups.google.com/forum/#!topic/binary-transparency/f-BI4o8HZW0
 for better integrity guarantees when getting updates.
 
 Boulder requires an installation of RabbitMQ, libtool-ltdl, goose, and
-MariaDB 10 to work correctly. On Ubuntu and CentOS, you may have to
+MariaDB 10.1 to work correctly. On Ubuntu and CentOS, you may have to
 install RabbitMQ from https://rabbitmq.com/download.html to get a
 recent version.
 

docker-compose.yml

@@ -1,27 +1,40 @@
 boulder:
     build: .
     dockerfile: Dockerfile
+    volumes:
+      # Cache built .a files for faster repeat runs
+      - /go/pkg/
+      - /tmp:/tmp
     net: bridge
+    extra_hosts:
+      - le.wtf:127.0.0.1
+      - boulder:127.0.0.1
     ports:
-      - 4000:4000
-      - 4002:4002
-      - 4003:4003
+      - 4000:4000 # ACME
+      - 4002:4002 # OCSP
+      - 4003:4003 # OCSP
+      - 4500:4500 # ct-test-srv
+      - 8000:8000 # debug ports
+      - 8001:8001
+      - 8002:8002
+      - 8003:8003
+      - 8004:8004
+      - 8055:8055 # dns-test-srv updates
+      - 9380:9380 # mail-test-srv
+      - 9381:9381 # mail-test-srv
     links:
       - bmysql:boulder-mysql
       - brabbitmq:boulder-rabbitmq
-    extra_hosts:
-      - boulder:127.0.0.1
 bmysql:
-    container_name: boulder-mysql
-    image: mariadb:10.0
+    image: mariadb:10.1
     net: bridge
     environment:
         MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
     command: mysqld --bind-address=0.0.0.0
+    log_driver: none
 brabbitmq:
-    container_name: boulder-rabbitmq
     image: rabbitmq:3
     net: bridge
     environment:
         RABBITMQ_NODE_IP_ADDRESS: "0.0.0.0"
-    log_driver: "none"
+    log_driver: none

reloader/reloader.go

@@ -23,6 +23,9 @@ func (r *Reloader) Stop() {
 	r.stopChan <- struct{}{}
 }
 
+// A pointer we can override for testing.
+var readFile = ioutil.ReadFile
+
 // New loads the filename provided, and calls the callback.  It then spawns a
 // goroutine to check for updates to that file, calling the callback again with
 // any new contents. The first load, and the first call to callback, are run
@@ -37,7 +40,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
 	if err != nil {
 		return nil, err
 	}
-	b, err := ioutil.ReadFile(filename)
+	b, err := readFile(filename)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +61,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
 				if !currentFileInfo.ModTime().After(fileInfo.ModTime()) {
 					continue
 				}
-				b, err := ioutil.ReadFile(filename)
+				b, err := readFile(filename)
 				if err != nil {
 					errorCallback(err)
 					continue

reloader/reloader_test.go

@@ -36,14 +36,16 @@ func TestNoStat(t *testing.T) {
 func TestNoRead(t *testing.T) {
 	f, _ := ioutil.TempFile("", "test-no-read.txt")
 	defer os.Remove(f.Name())
-	err := f.Chmod(0)
-	if err != nil {
-		t.Fatalf("failed to chmod file: %s", err)
+	oldReadFile := readFile
+	readFile = func(string) ([]byte, error) {
+		return nil, fmt.Errorf("read failed")
 	}
-	_, err = New(f.Name(), noop, testErrCb(t))
+	_, err := New(f.Name(), noop, testErrCb(t))
 	if err == nil {
 		t.Fatalf("Expected New to return error when permission denied.")
+		readFile = oldReadFile
 	}
+	readFile = oldReadFile
 }
 
 func TestFirstError(t *testing.T) {
@@ -182,10 +184,11 @@ func TestReloadFailure(t *testing.T) {
 
 	time.Sleep(15 * time.Millisecond)
 	// Create a file with no permissions
-	err = ioutil.WriteFile(filename, []byte("second body"), 0)
-	if err != nil {
-		t.Fatal(err)
+	oldReadFile := readFile
+	readFile = func(string) ([]byte, error) {
+		return nil, fmt.Errorf("permisssion denied")
 	}
+
 	fakeTick <- time.Now()
 	select {
 	case r := <-reloads:
@@ -195,11 +198,8 @@ func TestReloadFailure(t *testing.T) {
 	case <-time.After(5 * time.Second):
 		t.Fatalf("timed out waiting for reload")
 	}
+	readFile = oldReadFile
 
-	err = os.Remove(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
 	err = ioutil.WriteFile(filename, []byte("third body"), 0644)
 	if err != nil {
 		t.Fatal(err)

test.sh

@@ -72,15 +72,6 @@ function die() {
   exit 1
 }
 
-function build_certbot() {
-  run git clone \
-    https://www.github.com/certbot/certbot.git \
-    $CERTBOT_PATH || exit 1
-  cd $CERTBOT_PATH
-  run ./tools/venv.sh
-  cd -
-}
-
 function run_unit_tests() {
   if [ "${TRAVIS}" == "true" ]; then
 
@@ -188,18 +179,20 @@ if [[ "$RUN" =~ "integration" ]] ; then
   start_context "integration"
 
   if [ -z "$CERTBOT_PATH" ]; then
-    export CERTBOT_PATH=$(mktemp -d -t leXXXX)
+    export CERTBOT_PATH=$(mktemp -d -t cbpXXXX)
     echo "------------------------------------------------"
     echo "--- Checking out letsencrypt client is slow. ---"
     echo "--- Recommend setting \$CERTBOT_PATH to  ---"
     echo "--- client repo with initialized virtualenv  ---"
     echo "------------------------------------------------"
-    build_certbot
-  elif [ ! -d "${CERTBOT_PATH}" ]; then
-    build_certbot
+    run git clone \
+      https://www.github.com/certbot/certbot.git \
+      $CERTBOT_PATH || exit 1
   fi
 
-  source ${CERTBOT_PATH}/venv/bin/activate
+  if ! type certbot >/dev/null 2>/dev/null; then
+    source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate
+  fi
 
   python test/integration-test.py --all
   if [ "$?" != 0 ]; then

test/boulder-tools/Dockerfile

@@ -0,0 +1,4 @@
+FROM golang:1.5
+
+ADD build.sh /tmp/build.sh
+RUN bash /tmp/build.sh

test/boulder-tools/build.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -ex
+
+# Boulder deps
+apt-get update
+apt-get install -y --no-install-recommends apt-transport-https ca-certificates
+
+curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
+cat >/etc/apt/sources.list.d/bouldertools.list <<EOAPT
+deb https://deb.nodesource.com/node_4.x trusty main
+deb-src https://deb.nodesource.com/node_4.x trusty main
+deb http://ftp.debian.org/debian jessie-backports main
+EOAPT
+apt-get update
+apt-get install -y --no-install-recommends  -t jessie-backports letsencrypt python-letsencrypt-apache
+
+apt-get install -y --no-install-recommends \
+  libltdl-dev \
+  mariadb-client-core-10.0 \
+  nodejs \
+  rpm \
+  ruby \
+  ruby-dev \
+  rsyslog \
+  softhsm \
+  protobuf-compiler &
+
+# Install port forwarder, database migration tool, and testing tools.
+GOBIN=/usr/local/bin GOPATH=/tmp/gopath go get \
+  github.com/jsha/listenbuddy \
+  bitbucket.org/liamstask/goose/cmd/goose \
+  github.com/golang/lint/golint \
+  github.com/golang/mock/mockgen \
+  github.com/golang/protobuf/proto \
+  github.com/golang/protobuf/protoc-gen-go \
+  github.com/kisielk/errcheck \
+  github.com/mattn/goveralls \
+  github.com/modocache/gover \
+  github.com/tools/godep \
+  golang.org/x/tools/cmd/stringer \
+  golang.org/x/tools/cover &
+
+wait
+
+gem install fpm
+
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

test/certbot

@@ -0,0 +1,4 @@
+#!/bin/bash
+#
+# Temporary shim until the letsencrypt Debian package ships `certbot`
+exec letsencrypt "$@"

test/create_db.sh

@@ -19,9 +19,6 @@ fi
 # to the format we use in production, MIXED.
 mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';"
 
-# Drop all users to get a fresh start
-mysql $dbconn < test/drop_users.sql
-
 for dbenv in $DBENVS; do
   (
   db="boulder_sa_${dbenv}"

test/ct-test-srv/main.go

@@ -142,7 +142,7 @@ func main() {
 
 	is := integrationSrv{key: key}
 	s := &http.Server{
-		Addr:    "localhost:4500",
+		Addr:    "0.0.0.0:4500",
 		Handler: http.HandlerFunc(is.handler),
 	}
 	log.Fatal(s.ListenAndServe())

test/docker-environment

@@ -0,0 +1,4 @@
+PATH=/go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
+GOPATH=/go
+GOBIN=/go/src/github.com/letsencrypt/boulder/bin
+GO15VENDOREXPERIMENT=1

test/drop_users.sql

@@ -3,6 +3,9 @@
 -- Note that dropping a non-existing user produces an error that aborts the
 -- script, so we first grant a harmless privilege to each user to ensure it
 -- exists.
+
+USE mysql;
+
 GRANT USAGE ON *.* TO 'policy'@'localhost';
 DROP USER 'policy'@'localhost';
 GRANT USAGE ON *.* TO 'sa'@'localhost';
@@ -21,3 +24,5 @@ GRANT USAGE ON *.* TO 'cert_checker'@'localhost';
 DROP USER 'cert_checker'@'localhost';
 GRANT USAGE ON *.* TO 'backfiller'@'localhost';
 DROP USER 'backfiller'@'localhost';
+GRANT USAGE ON *.* TO 'test_setup'@'localhost';
+DROP USER 'test_setup'@'localhost';

test/entrypoint.sh

@@ -28,10 +28,14 @@ wait_tcp_port boulder-rabbitmq 5672
 MYSQL_CONTAINER=1 $DIR/create_db.sh
 
 # Set up rabbitmq exchange
-go run cmd/rabbitmq-setup/main.go -server amqp://boulder-rabbitmq
+rabbitmq-setup -server amqp://boulder-rabbitmq
 
 if [[ $# -eq 0 ]]; then
     exec ./start.py
 fi
 
+# TODO(jsha): Change to an unprivileged user before running commands. Currently,
+# running as an unprivileged user causes the certbot integration test to fail
+# during the test of the manual plugin. There's a call to killpg in there that
+# kills the whole test, but only when run under `su buser -c "..."`
 exec $@

test/mail-test-srv/main.go

@@ -16,7 +16,7 @@ import (
 	blog "github.com/letsencrypt/boulder/log"
 )
 
-var apiPort = flag.String("http", "9381", "http port to listen on")
+var listenAPI = flag.String("http", "0.0.0.0:9381", "http port to listen on")
 
 type rcvdMail struct {
 	From string
@@ -162,7 +162,7 @@ func serveSMTP(l net.Listener) error {
 }
 
 func main() {
-	l, err := net.Listen("tcp", ":9380")
+	l, err := net.Listen("tcp", "0.0.0.0:9380")
 	if err != nil {
 		log.Fatalln("Couldn't bind for SMTP", err)
 	}
@@ -170,7 +170,7 @@ func main() {
 
 	setupHTTP(http.DefaultServeMux)
 	go func() {
-		err := http.ListenAndServe(":"+*apiPort, http.DefaultServeMux)
+		err := http.ListenAndServe(*listenAPI, http.DefaultServeMux)
 		if err != nil {
 			log.Fatalln("Couldn't start HTTP server", err)
 		}

test/run-docker.sh

@@ -36,7 +36,7 @@ if [[ "$(is_running boulder-mysql)" != "true" ]]; then
 	docker run -d \
 		-e MYSQL_ALLOW_EMPTY_PASSWORD=yes \
 		--name boulder-mysql \
-		mariadb:10.0 mysqld --bind-address=0.0.0.0
+		mariadb:10.1 mysqld --bind-address=0.0.0.0
 fi
 
 if [[ "$(is_running boulder-rabbitmq)" != "true" ]]; then

test/sa_db_users.sql

@@ -14,6 +14,18 @@
 -- drop command will fail. So we grant the dummy `USAGE` privilege to make sure
 -- the user exists and then drop the user.
 
+
+-- These lines require MariaDB 10.1
+CREATE USER IF NOT EXISTS 'policy'@'localhost';
+CREATE USER IF NOT EXISTS 'sa'@'localhost';
+CREATE USER IF NOT EXISTS 'ocsp_resp'@'localhost';
+CREATE USER IF NOT EXISTS 'revoker'@'localhost';
+CREATE USER IF NOT EXISTS 'importer'@'localhost';
+CREATE USER IF NOT EXISTS 'mailer'@'localhost';
+CREATE USER IF NOT EXISTS 'cert_checker'@'localhost';
+CREATE USER IF NOT EXISTS 'ocsp_update'@'localhost';
+CREATE USER IF NOT EXISTS 'test_setup'@'localhost';
+
 -- Storage Authority
 GRANT SELECT,INSERT,UPDATE ON authz TO 'sa'@'localhost';
 GRANT SELECT,INSERT,UPDATE,DELETE ON pendingAuthorizations TO 'sa'@'localhost';
@@ -55,9 +67,5 @@ GRANT SELECT ON fqdnSets TO 'mailer'@'localhost';
 -- Cert checker
 GRANT SELECT ON certificates TO 'cert_checker'@'localhost';
 
--- Name set table backfiller
-GRANT SELECT ON certificates to 'backfiller'@'localhost';
-GRANT INSERT,SELECT ON fqdnSets to 'backfiller'@'localhost';
-
 -- Test setup and teardown
 GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';