diff --git a/.dockerignore b/.dockerignore index 07354dc7e..7fcd950a0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,2 @@ bin tags -.git -test/js diff --git a/.travis.yml b/.travis.yml index 3e6fef536..6ad39695a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,23 +9,11 @@ addons: - boulder - boulder-mysql - boulder-rabbitmq - apt: - packages: - - lsb-release - - python-dev - - python-virtualenv - - gcc - - libaugeas0 - - libssl-dev - - libffi-dev - - ca-certificates - - rsyslog - mariadb: "10.0" -sudo: false +sudo: required services: - - rabbitmq + - docker matrix: fast_finish: true @@ -43,13 +31,6 @@ branches: - release - /^test-.*$/ -# By providing our own install command we avoid Travis' default Go install -# command, which runs `go get`. We specifically want to avoid that because we -# want to ensure all our dependencies are vendored. -install: - - travis_retry test/travis-before-install.sh - - cd $GOPATH/src/github.com/letsencrypt/boulder - env: global: - PATH=$HOME/bin:$PATH # protoc gets installed here @@ -61,6 +42,10 @@ env: - RUN="integration" BOULDER_CONFIG="test/boulder-config-next.json" - RUN="unit" -script: - - bash test.sh +install: + - docker-compose pull + - docker pull letsencrypt/boulder-tools + - docker-compose build +script: + - docker-compose run -e RUN="${RUN}" -e TRAVIS="${TRAVIS}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" boulder ./test.sh diff --git a/Dockerfile b/Dockerfile index 67829bd13..7abed4f85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,24 @@ -FROM golang:1.5 - -MAINTAINER J.C. Jones "jjones@letsencrypt.org" -MAINTAINER William Budington "bill@eff.org" - -# Install dependencies packages -RUN apt-get update && apt-get install -y \ - libltdl-dev \ - mariadb-client-core-10.0 \ - nodejs \ - rsyslog \ - softhsm \ - --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Install port forwarder, database migration tool and go lint -RUN go get -v \ - github.com/jsha/listenbuddy \ - bitbucket.org/liamstask/goose/cmd/goose \ - github.com/golang/lint/golint +FROM letsencrypt/boulder-tools:latest # Boulder exposes its web application at port TCP 4000 EXPOSE 4000 4002 4003 8053 8055 ENV GO15VENDOREXPERIMENT 1 +ENV GOBIN /go/src/github.com/letsencrypt/boulder/bin +ENV PATH /go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/ +ENV GOPATH /go + +RUN adduser --disabled-password --gecos "" --home /go/src/github.com/letsencrypt/boulder -q buser +RUN chown -R buser /go/ WORKDIR /go/src/github.com/letsencrypt/boulder -ENTRYPOINT [ "./test/entrypoint.sh" ] - # Copy in the Boulder sources -COPY . /go/src/github.com/letsencrypt/boulder +COPY . . +RUN mkdir bin +RUN go install ./cmd/rabbitmq-setup +COPY ./test/certbot /go/bin/ -RUN GOBIN=/go/src/github.com/letsencrypt/boulder/bin go install ./... +RUN chown -R buser /go/ + +ENTRYPOINT [ "./test/entrypoint.sh" ] diff --git a/README.md b/README.md index 2712981cf..3a2936e7d 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ setting](https://groups.google.com/forum/#!topic/binary-transparency/f-BI4o8HZW0 for better integrity guarantees when getting updates. Boulder requires an installation of RabbitMQ, libtool-ltdl, goose, and -MariaDB 10 to work correctly. On Ubuntu and CentOS, you may have to +MariaDB 10.1 to work correctly. On Ubuntu and CentOS, you may have to install RabbitMQ from https://rabbitmq.com/download.html to get a recent version. diff --git a/docker-compose.yml b/docker-compose.yml index 709c21461..8e507547e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,27 +1,42 @@ boulder: build: . dockerfile: Dockerfile + environment: + FAKE_DNS: 127.0.0.1 + volumes: + # Cache built .a files for faster repeat runs + - /go/pkg/ + - /tmp:/tmp net: bridge + extra_hosts: + - le.wtf:127.0.0.1 + - boulder:127.0.0.1 ports: - - 4000:4000 - - 4002:4002 - - 4003:4003 + - 4000:4000 # ACME + - 4002:4002 # OCSP + - 4003:4003 # OCSP + - 4500:4500 # ct-test-srv + - 8000:8000 # debug ports + - 8001:8001 + - 8002:8002 + - 8003:8003 + - 8004:8004 + - 8055:8055 # dns-test-srv updates + - 9380:9380 # mail-test-srv + - 9381:9381 # mail-test-srv links: - bmysql:boulder-mysql - brabbitmq:boulder-rabbitmq - extra_hosts: - - boulder:127.0.0.1 bmysql: - container_name: boulder-mysql - image: mariadb:10.0 + image: mariadb:10.1 net: bridge environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" command: mysqld --bind-address=0.0.0.0 + log_driver: none brabbitmq: - container_name: boulder-rabbitmq image: rabbitmq:3 net: bridge environment: RABBITMQ_NODE_IP_ADDRESS: "0.0.0.0" - log_driver: "none" + log_driver: none diff --git a/reloader/reloader.go b/reloader/reloader.go index 751ff559b..d885af630 100644 --- a/reloader/reloader.go +++ b/reloader/reloader.go @@ -23,6 +23,9 @@ func (r *Reloader) Stop() { r.stopChan <- struct{}{} } +// A pointer we can override for testing. +var readFile = ioutil.ReadFile + // New loads the filename provided, and calls the callback. It then spawns a // goroutine to check for updates to that file, calling the callback again with // any new contents. The first load, and the first call to callback, are run @@ -37,7 +40,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er if err != nil { return nil, err } - b, err := ioutil.ReadFile(filename) + b, err := readFile(filename) if err != nil { return nil, err } @@ -58,7 +61,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er if !currentFileInfo.ModTime().After(fileInfo.ModTime()) { continue } - b, err := ioutil.ReadFile(filename) + b, err := readFile(filename) if err != nil { errorCallback(err) continue diff --git a/reloader/reloader_test.go b/reloader/reloader_test.go index 66bd82910..7aef8926a 100644 --- a/reloader/reloader_test.go +++ b/reloader/reloader_test.go @@ -36,14 +36,16 @@ func TestNoStat(t *testing.T) { func TestNoRead(t *testing.T) { f, _ := ioutil.TempFile("", "test-no-read.txt") defer os.Remove(f.Name()) - err := f.Chmod(0) - if err != nil { - t.Fatalf("failed to chmod file: %s", err) + oldReadFile := readFile + readFile = func(string) ([]byte, error) { + return nil, fmt.Errorf("read failed") } - _, err = New(f.Name(), noop, testErrCb(t)) + _, err := New(f.Name(), noop, testErrCb(t)) if err == nil { t.Fatalf("Expected New to return error when permission denied.") + readFile = oldReadFile } + readFile = oldReadFile } func TestFirstError(t *testing.T) { @@ -182,10 +184,11 @@ func TestReloadFailure(t *testing.T) { time.Sleep(15 * time.Millisecond) // Create a file with no permissions - err = ioutil.WriteFile(filename, []byte("second body"), 0) - if err != nil { - t.Fatal(err) + oldReadFile := readFile + readFile = func(string) ([]byte, error) { + return nil, fmt.Errorf("permisssion denied") } + fakeTick <- time.Now() select { case r := <-reloads: @@ -195,11 +198,8 @@ func TestReloadFailure(t *testing.T) { case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for reload") } + readFile = oldReadFile - err = os.Remove(filename) - if err != nil { - t.Fatal(err) - } err = ioutil.WriteFile(filename, []byte("third body"), 0644) if err != nil { t.Fatal(err) diff --git a/test.sh b/test.sh index bb3f7c35a..6528ea02b 100755 --- a/test.sh +++ b/test.sh @@ -72,15 +72,6 @@ function die() { exit 1 } -function build_certbot() { - run git clone \ - https://www.github.com/certbot/certbot.git \ - $CERTBOT_PATH || exit 1 - cd $CERTBOT_PATH - run ./tools/venv.sh - cd - -} - function run_unit_tests() { if [ "${TRAVIS}" == "true" ]; then @@ -188,18 +179,20 @@ if [[ "$RUN" =~ "integration" ]] ; then start_context "integration" if [ -z "$CERTBOT_PATH" ]; then - export CERTBOT_PATH=$(mktemp -d -t leXXXX) + export CERTBOT_PATH=$(mktemp -d -t cbpXXXX) echo "------------------------------------------------" echo "--- Checking out letsencrypt client is slow. ---" echo "--- Recommend setting \$CERTBOT_PATH to ---" echo "--- client repo with initialized virtualenv ---" echo "------------------------------------------------" - build_certbot - elif [ ! -d "${CERTBOT_PATH}" ]; then - build_certbot + run git clone \ + https://www.github.com/certbot/certbot.git \ + $CERTBOT_PATH || exit 1 fi - source ${CERTBOT_PATH}/venv/bin/activate + if ! type certbot >/dev/null 2>/dev/null; then + source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate + fi python test/integration-test.py --all if [ "$?" != 0 ]; then diff --git a/test/certbot b/test/certbot new file mode 100755 index 000000000..0a4f4c4cc --- /dev/null +++ b/test/certbot @@ -0,0 +1,4 @@ +#!/bin/bash +# +# Temporary shim until the letsencrypt Debian package ships `certbot` +exec letsencrypt "$@" diff --git a/test/create_db.sh b/test/create_db.sh index 030304976..d4fcd7b46 100755 --- a/test/create_db.sh +++ b/test/create_db.sh @@ -19,9 +19,6 @@ fi # to the format we use in production, MIXED. mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';" -# Drop all users to get a fresh start -mysql $dbconn < test/drop_users.sql - for dbenv in $DBENVS; do ( db="boulder_sa_${dbenv}" diff --git a/test/ct-test-srv/main.go b/test/ct-test-srv/main.go index b08db8fe1..1dbd8fc65 100644 --- a/test/ct-test-srv/main.go +++ b/test/ct-test-srv/main.go @@ -142,7 +142,7 @@ func main() { is := integrationSrv{key: key} s := &http.Server{ - Addr: "localhost:4500", + Addr: "0.0.0.0:4500", Handler: http.HandlerFunc(is.handler), } log.Fatal(s.ListenAndServe()) diff --git a/test/docker-environment b/test/docker-environment new file mode 100644 index 000000000..ed7e00978 --- /dev/null +++ b/test/docker-environment @@ -0,0 +1,4 @@ +PATH=/go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/ +GOPATH=/go +GOBIN=/go/src/github.com/letsencrypt/boulder/bin +GO15VENDOREXPERIMENT=1 diff --git a/test/drop_users.sql b/test/drop_users.sql index 81be0534e..d00bec5aa 100644 --- a/test/drop_users.sql +++ b/test/drop_users.sql @@ -3,6 +3,9 @@ -- Note that dropping a non-existing user produces an error that aborts the -- script, so we first grant a harmless privilege to each user to ensure it -- exists. + +USE mysql; + GRANT USAGE ON *.* TO 'policy'@'localhost'; DROP USER 'policy'@'localhost'; GRANT USAGE ON *.* TO 'sa'@'localhost'; @@ -21,3 +24,5 @@ GRANT USAGE ON *.* TO 'cert_checker'@'localhost'; DROP USER 'cert_checker'@'localhost'; GRANT USAGE ON *.* TO 'backfiller'@'localhost'; DROP USER 'backfiller'@'localhost'; +GRANT USAGE ON *.* TO 'test_setup'@'localhost'; +DROP USER 'test_setup'@'localhost'; diff --git a/test/entrypoint.sh b/test/entrypoint.sh index e53b0b509..d7d04bf45 100755 --- a/test/entrypoint.sh +++ b/test/entrypoint.sh @@ -28,10 +28,14 @@ wait_tcp_port boulder-rabbitmq 5672 MYSQL_CONTAINER=1 $DIR/create_db.sh # Set up rabbitmq exchange -go run cmd/rabbitmq-setup/main.go -server amqp://boulder-rabbitmq +rabbitmq-setup -server amqp://boulder-rabbitmq if [[ $# -eq 0 ]]; then exec ./start.py fi +# TODO(jsha): Change to an unprivileged user before running commands. Currently, +# running as an unprivileged user causes the certbot integration test to fail +# during the test of the manual plugin. There's a call to killpg in there that +# kills the whole test, but only when run under `su buser -c "..."` exec $@ diff --git a/test/mail-test-srv/main.go b/test/mail-test-srv/main.go index 74ee0f314..75da93027 100644 --- a/test/mail-test-srv/main.go +++ b/test/mail-test-srv/main.go @@ -16,7 +16,7 @@ import ( blog "github.com/letsencrypt/boulder/log" ) -var apiPort = flag.String("http", "9381", "http port to listen on") +var listenAPI = flag.String("http", "0.0.0.0:9381", "http port to listen on") type rcvdMail struct { From string @@ -162,7 +162,7 @@ func serveSMTP(l net.Listener) error { } func main() { - l, err := net.Listen("tcp", ":9380") + l, err := net.Listen("tcp", "0.0.0.0:9380") if err != nil { log.Fatalln("Couldn't bind for SMTP", err) } @@ -170,7 +170,7 @@ func main() { setupHTTP(http.DefaultServeMux) go func() { - err := http.ListenAndServe(":"+*apiPort, http.DefaultServeMux) + err := http.ListenAndServe(*listenAPI, http.DefaultServeMux) if err != nil { log.Fatalln("Couldn't start HTTP server", err) } diff --git a/test/run-docker.sh b/test/run-docker.sh index 0530e2b6b..0e8492def 100755 --- a/test/run-docker.sh +++ b/test/run-docker.sh @@ -36,7 +36,7 @@ if [[ "$(is_running boulder-mysql)" != "true" ]]; then docker run -d \ -e MYSQL_ALLOW_EMPTY_PASSWORD=yes \ --name boulder-mysql \ - mariadb:10.0 mysqld --bind-address=0.0.0.0 + mariadb:10.1 mysqld --bind-address=0.0.0.0 fi if [[ "$(is_running boulder-rabbitmq)" != "true" ]]; then diff --git a/test/sa_db_users.sql b/test/sa_db_users.sql index 73c525936..5361141d5 100644 --- a/test/sa_db_users.sql +++ b/test/sa_db_users.sql @@ -14,6 +14,18 @@ -- drop command will fail. So we grant the dummy `USAGE` privilege to make sure -- the user exists and then drop the user. + +-- These lines require MariaDB 10.1 +CREATE USER IF NOT EXISTS 'policy'@'localhost'; +CREATE USER IF NOT EXISTS 'sa'@'localhost'; +CREATE USER IF NOT EXISTS 'ocsp_resp'@'localhost'; +CREATE USER IF NOT EXISTS 'revoker'@'localhost'; +CREATE USER IF NOT EXISTS 'importer'@'localhost'; +CREATE USER IF NOT EXISTS 'mailer'@'localhost'; +CREATE USER IF NOT EXISTS 'cert_checker'@'localhost'; +CREATE USER IF NOT EXISTS 'ocsp_update'@'localhost'; +CREATE USER IF NOT EXISTS 'test_setup'@'localhost'; + -- Storage Authority GRANT SELECT,INSERT,UPDATE ON authz TO 'sa'@'localhost'; GRANT SELECT,INSERT,UPDATE,DELETE ON pendingAuthorizations TO 'sa'@'localhost'; @@ -55,9 +67,5 @@ GRANT SELECT ON fqdnSets TO 'mailer'@'localhost'; -- Cert checker GRANT SELECT ON certificates TO 'cert_checker'@'localhost'; --- Name set table backfiller -GRANT SELECT ON certificates to 'backfiller'@'localhost'; -GRANT INSERT,SELECT ON fqdnSets to 'backfiller'@'localhost'; - -- Test setup and teardown GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';