From b954dcc010e32589f5f60695bf28d3cb68e96167 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 20 May 2016 15:57:10 -0700 Subject: [PATCH] Revert "Run Travis tests in Docker (#1830)" (#1834) This reverts commit 92d94f2 and commit 0b4623f to unbreak the Certbot build. --- .dockerignore | 2 + .travis.yml | 31 +++++++++---- Dockerfile | 40 ++++++++++------- README.md | 2 +- docker-compose.yml | 31 ++++--------- reloader/reloader.go | 7 +-- reloader/reloader_test.go | 22 +++++----- test.sh | 21 ++++++--- test/boulder-tools/Dockerfile | 4 -- test/boulder-tools/build.sh | 46 ------------------- test/certbot | 4 -- test/create_db.sh | 5 ++- test/ct-test-srv/main.go | 2 +- test/docker-environment | 4 -- test/drop_users.sql | 5 --- test/entrypoint.sh | 6 +-- test/mail-test-srv/main.go | 6 +-- test/mariadb100_users.sql | 83 ----------------------------------- test/run-docker.sh | 2 +- test/sa_db_users.sql | 16 ++----- 20 files changed, 100 insertions(+), 239 deletions(-) delete mode 100644 test/boulder-tools/Dockerfile delete mode 100644 test/boulder-tools/build.sh delete mode 100755 test/certbot delete mode 100644 test/docker-environment delete mode 100644 test/mariadb100_users.sql diff --git a/.dockerignore b/.dockerignore index 7fcd950a0..07354dc7e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ bin tags +.git +test/js diff --git a/.travis.yml b/.travis.yml index 6ad39695a..3e6fef536 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,23 @@ addons: - boulder - boulder-mysql - boulder-rabbitmq + apt: + packages: + - lsb-release + - python-dev + - python-virtualenv + - gcc + - libaugeas0 + - libssl-dev + - libffi-dev + - ca-certificates + - rsyslog + mariadb: "10.0" -sudo: required +sudo: false services: - - docker + - rabbitmq matrix: fast_finish: true @@ -31,6 +43,13 @@ branches: - release - /^test-.*$/ +# By providing our own install command we avoid Travis' default Go install +# command, which runs `go get`. We specifically want to avoid that because we +# want to ensure all our dependencies are vendored. +install: + - travis_retry test/travis-before-install.sh + - cd $GOPATH/src/github.com/letsencrypt/boulder + env: global: - PATH=$HOME/bin:$PATH # protoc gets installed here @@ -42,10 +61,6 @@ env: - RUN="integration" BOULDER_CONFIG="test/boulder-config-next.json" - RUN="unit" -install: - - docker-compose pull - - docker pull letsencrypt/boulder-tools - - docker-compose build - script: - - docker-compose run -e RUN="${RUN}" -e TRAVIS="${TRAVIS}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" boulder ./test.sh + - bash test.sh + diff --git a/Dockerfile b/Dockerfile index 7abed4f85..67829bd13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,34 @@ -FROM letsencrypt/boulder-tools:latest +FROM golang:1.5 + +MAINTAINER J.C. Jones "jjones@letsencrypt.org" +MAINTAINER William Budington "bill@eff.org" + +# Install dependencies packages +RUN apt-get update && apt-get install -y \ + libltdl-dev \ + mariadb-client-core-10.0 \ + nodejs \ + rsyslog \ + softhsm \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install port forwarder, database migration tool and go lint +RUN go get -v \ + github.com/jsha/listenbuddy \ + bitbucket.org/liamstask/goose/cmd/goose \ + github.com/golang/lint/golint # Boulder exposes its web application at port TCP 4000 EXPOSE 4000 4002 4003 8053 8055 ENV GO15VENDOREXPERIMENT 1 -ENV GOBIN /go/src/github.com/letsencrypt/boulder/bin -ENV PATH /go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/ -ENV GOPATH /go - -RUN adduser --disabled-password --gecos "" --home /go/src/github.com/letsencrypt/boulder -q buser -RUN chown -R buser /go/ WORKDIR /go/src/github.com/letsencrypt/boulder -# Copy in the Boulder sources -COPY . . -RUN mkdir bin -RUN go install ./cmd/rabbitmq-setup -COPY ./test/certbot /go/bin/ - -RUN chown -R buser /go/ - ENTRYPOINT [ "./test/entrypoint.sh" ] + +# Copy in the Boulder sources +COPY . /go/src/github.com/letsencrypt/boulder + +RUN GOBIN=/go/src/github.com/letsencrypt/boulder/bin go install ./... diff --git a/README.md b/README.md index 3a2936e7d..2712981cf 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ setting](https://groups.google.com/forum/#!topic/binary-transparency/f-BI4o8HZW0 for better integrity guarantees when getting updates. Boulder requires an installation of RabbitMQ, libtool-ltdl, goose, and -MariaDB 10.1 to work correctly. On Ubuntu and CentOS, you may have to +MariaDB 10 to work correctly. On Ubuntu and CentOS, you may have to install RabbitMQ from https://rabbitmq.com/download.html to get a recent version. diff --git a/docker-compose.yml b/docker-compose.yml index 077201155..709c21461 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,27 @@ boulder: build: . dockerfile: Dockerfile - volumes: - # Cache built .a files for faster repeat runs - - /go/pkg/ - - /tmp:/tmp net: bridge - extra_hosts: - - le.wtf:127.0.0.1 - - boulder:127.0.0.1 ports: - - 4000:4000 # ACME - - 4002:4002 # OCSP - - 4003:4003 # OCSP - - 4500:4500 # ct-test-srv - - 8000:8000 # debug ports - - 8001:8001 - - 8002:8002 - - 8003:8003 - - 8004:8004 - - 8055:8055 # dns-test-srv updates - - 9380:9380 # mail-test-srv - - 9381:9381 # mail-test-srv + - 4000:4000 + - 4002:4002 + - 4003:4003 links: - bmysql:boulder-mysql - brabbitmq:boulder-rabbitmq + extra_hosts: + - boulder:127.0.0.1 bmysql: - image: mariadb:10.1 + container_name: boulder-mysql + image: mariadb:10.0 net: bridge environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" command: mysqld --bind-address=0.0.0.0 - log_driver: none brabbitmq: + container_name: boulder-rabbitmq image: rabbitmq:3 net: bridge environment: RABBITMQ_NODE_IP_ADDRESS: "0.0.0.0" - log_driver: none + log_driver: "none" diff --git a/reloader/reloader.go b/reloader/reloader.go index d885af630..751ff559b 100644 --- a/reloader/reloader.go +++ b/reloader/reloader.go @@ -23,9 +23,6 @@ func (r *Reloader) Stop() { r.stopChan <- struct{}{} } -// A pointer we can override for testing. -var readFile = ioutil.ReadFile - // New loads the filename provided, and calls the callback. It then spawns a // goroutine to check for updates to that file, calling the callback again with // any new contents. The first load, and the first call to callback, are run @@ -40,7 +37,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er if err != nil { return nil, err } - b, err := readFile(filename) + b, err := ioutil.ReadFile(filename) if err != nil { return nil, err } @@ -61,7 +58,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er if !currentFileInfo.ModTime().After(fileInfo.ModTime()) { continue } - b, err := readFile(filename) + b, err := ioutil.ReadFile(filename) if err != nil { errorCallback(err) continue diff --git a/reloader/reloader_test.go b/reloader/reloader_test.go index 7aef8926a..66bd82910 100644 --- a/reloader/reloader_test.go +++ b/reloader/reloader_test.go @@ -36,16 +36,14 @@ func TestNoStat(t *testing.T) { func TestNoRead(t *testing.T) { f, _ := ioutil.TempFile("", "test-no-read.txt") defer os.Remove(f.Name()) - oldReadFile := readFile - readFile = func(string) ([]byte, error) { - return nil, fmt.Errorf("read failed") + err := f.Chmod(0) + if err != nil { + t.Fatalf("failed to chmod file: %s", err) } - _, err := New(f.Name(), noop, testErrCb(t)) + _, err = New(f.Name(), noop, testErrCb(t)) if err == nil { t.Fatalf("Expected New to return error when permission denied.") - readFile = oldReadFile } - readFile = oldReadFile } func TestFirstError(t *testing.T) { @@ -184,11 +182,10 @@ func TestReloadFailure(t *testing.T) { time.Sleep(15 * time.Millisecond) // Create a file with no permissions - oldReadFile := readFile - readFile = func(string) ([]byte, error) { - return nil, fmt.Errorf("permisssion denied") + err = ioutil.WriteFile(filename, []byte("second body"), 0) + if err != nil { + t.Fatal(err) } - fakeTick <- time.Now() select { case r := <-reloads: @@ -198,8 +195,11 @@ func TestReloadFailure(t *testing.T) { case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for reload") } - readFile = oldReadFile + err = os.Remove(filename) + if err != nil { + t.Fatal(err) + } err = ioutil.WriteFile(filename, []byte("third body"), 0644) if err != nil { t.Fatal(err) diff --git a/test.sh b/test.sh index 6528ea02b..bb3f7c35a 100755 --- a/test.sh +++ b/test.sh @@ -72,6 +72,15 @@ function die() { exit 1 } +function build_certbot() { + run git clone \ + https://www.github.com/certbot/certbot.git \ + $CERTBOT_PATH || exit 1 + cd $CERTBOT_PATH + run ./tools/venv.sh + cd - +} + function run_unit_tests() { if [ "${TRAVIS}" == "true" ]; then @@ -179,20 +188,18 @@ if [[ "$RUN" =~ "integration" ]] ; then start_context "integration" if [ -z "$CERTBOT_PATH" ]; then - export CERTBOT_PATH=$(mktemp -d -t cbpXXXX) + export CERTBOT_PATH=$(mktemp -d -t leXXXX) echo "------------------------------------------------" echo "--- Checking out letsencrypt client is slow. ---" echo "--- Recommend setting \$CERTBOT_PATH to ---" echo "--- client repo with initialized virtualenv ---" echo "------------------------------------------------" - run git clone \ - https://www.github.com/certbot/certbot.git \ - $CERTBOT_PATH || exit 1 + build_certbot + elif [ ! -d "${CERTBOT_PATH}" ]; then + build_certbot fi - if ! type certbot >/dev/null 2>/dev/null; then - source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate - fi + source ${CERTBOT_PATH}/venv/bin/activate python test/integration-test.py --all if [ "$?" != 0 ]; then diff --git a/test/boulder-tools/Dockerfile b/test/boulder-tools/Dockerfile deleted file mode 100644 index 35b988011..000000000 --- a/test/boulder-tools/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM golang:1.5 - -ADD build.sh /tmp/build.sh -RUN bash /tmp/build.sh diff --git a/test/boulder-tools/build.sh b/test/boulder-tools/build.sh deleted file mode 100644 index 51cb06ea8..000000000 --- a/test/boulder-tools/build.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -ex - -# Boulder deps -apt-get update -apt-get install -y --no-install-recommends apt-transport-https ca-certificates - -curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - -cat >/etc/apt/sources.list.d/bouldertools.list <