letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Run Travis tests in Docker (#1830)" (#1834) 

This reverts commit 92d94f2 and commit 0b4623f to unbreak the Certbot build.
This commit is contained in:
Jacob Hoffman-Andrews 2016-05-20 15:57:10 -07:00
parent 0b4623f8a3
commit b954dcc010
20 changed files with 100 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.dockerignore
View File

@ -1,2 +1,4 @@
bin
tags
.git
test/js

31
.travis.yml
View File

@ -9,11 +9,23 @@ addons:
- boulder
- boulder-mysql
- boulder-rabbitmq
apt:
packages:
- lsb-release
- python-dev
- python-virtualenv
- gcc
- libaugeas0
- libssl-dev
- libffi-dev
- ca-certificates
- rsyslog
mariadb: "10.0"
sudo: required
sudo: false
services:
- docker
- rabbitmq
matrix:
fast_finish: true
@ -31,6 +43,13 @@ branches:
- release
- /^test-.*$/
# By providing our own install command we avoid Travis' default Go install
# command, which runs `go get`. We specifically want to avoid that because we
# want to ensure all our dependencies are vendored.
install:
- travis_retry test/travis-before-install.sh
- cd $GOPATH/src/github.com/letsencrypt/boulder
env:
global:
- PATH=$HOME/bin:$PATH # protoc gets installed here
@ -42,10 +61,6 @@ env:
- RUN="integration" BOULDER_CONFIG="test/boulder-config-next.json"
- RUN="unit"
install:
- docker-compose pull
- docker pull letsencrypt/boulder-tools
- docker-compose build
script:
- docker-compose run -e RUN="${RUN}" -e TRAVIS="${TRAVIS}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" boulder ./test.sh
- bash test.sh

40
Dockerfile
View File

@ -1,24 +1,34 @@
FROM letsencrypt/boulder-tools:latest
FROM golang:1.5
MAINTAINER J.C. Jones "jjones@letsencrypt.org"
MAINTAINER William Budington "bill@eff.org"
# Install dependencies packages
RUN apt-get update && apt-get install -y \
libltdl-dev \
mariadb-client-core-10.0 \
nodejs \
rsyslog \
softhsm \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install port forwarder, database migration tool and go lint
RUN go get -v \
github.com/jsha/listenbuddy \
bitbucket.org/liamstask/goose/cmd/goose \
github.com/golang/lint/golint
# Boulder exposes its web application at port TCP 4000
EXPOSE 4000 4002 4003 8053 8055
ENV GO15VENDOREXPERIMENT 1
ENV GOBIN /go/src/github.com/letsencrypt/boulder/bin
ENV PATH /go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
ENV GOPATH /go
RUN adduser --disabled-password --gecos "" --home /go/src/github.com/letsencrypt/boulder -q buser
RUN chown -R buser /go/
WORKDIR /go/src/github.com/letsencrypt/boulder
# Copy in the Boulder sources
COPY . .
RUN mkdir bin
RUN go install ./cmd/rabbitmq-setup
COPY ./test/certbot /go/bin/
RUN chown -R buser /go/
ENTRYPOINT [ "./test/entrypoint.sh" ]
# Copy in the Boulder sources
COPY . /go/src/github.com/letsencrypt/boulder
RUN GOBIN=/go/src/github.com/letsencrypt/boulder/bin go install ./...

2
README.md
View File

@ -27,7 +27,7 @@ setting](https://groups.google.com/forum/#!topic/binary-transparency/f-BI4o8HZW0
for better integrity guarantees when getting updates.
Boulder requires an installation of RabbitMQ, libtool-ltdl, goose, and
MariaDB 10.1 to work correctly. On Ubuntu and CentOS, you may have to
MariaDB 10 to work correctly. On Ubuntu and CentOS, you may have to
install RabbitMQ from https://rabbitmq.com/download.html to get a
recent version.

31
docker-compose.yml
View File

@ -1,40 +1,27 @@
boulder:
build: .
dockerfile: Dockerfile
volumes:
# Cache built .a files for faster repeat runs
- /go/pkg/
- /tmp:/tmp
net: bridge
extra_hosts:
- le.wtf:127.0.0.1
- boulder:127.0.0.1
ports:
- 4000:4000 # ACME
- 4002:4002 # OCSP
- 4003:4003 # OCSP
- 4500:4500 # ct-test-srv
- 8000:8000 # debug ports
- 8001:8001
- 8002:8002
- 8003:8003
- 8004:8004
- 8055:8055 # dns-test-srv updates
- 9380:9380 # mail-test-srv
- 9381:9381 # mail-test-srv
- 4000:4000
- 4002:4002
- 4003:4003
links:
- bmysql:boulder-mysql
- brabbitmq:boulder-rabbitmq
extra_hosts:
- boulder:127.0.0.1
bmysql:
image: mariadb:10.1
container_name: boulder-mysql
image: mariadb:10.0
net: bridge
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
command: mysqld --bind-address=0.0.0.0
log_driver: none
brabbitmq:
container_name: boulder-rabbitmq
image: rabbitmq:3
net: bridge
environment:
RABBITMQ_NODE_IP_ADDRESS: "0.0.0.0"
log_driver: none
log_driver: "none"

7
reloader/reloader.go
View File

@ -23,9 +23,6 @@ func (r *Reloader) Stop() {
r.stopChan <- struct{}{}
}
// A pointer we can override for testing.
var readFile = ioutil.ReadFile
// New loads the filename provided, and calls the callback. It then spawns a
// goroutine to check for updates to that file, calling the callback again with
// any new contents. The first load, and the first call to callback, are run
@ -40,7 +37,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
if err != nil {
return nil, err
}
b, err := readFile(filename)
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
@ -61,7 +58,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
if !currentFileInfo.ModTime().After(fileInfo.ModTime()) {
continue
}
b, err := readFile(filename)
b, err := ioutil.ReadFile(filename)
if err != nil {
errorCallback(err)
continue

22
reloader/reloader_test.go
View File

@ -36,16 +36,14 @@ func TestNoStat(t *testing.T) {
func TestNoRead(t *testing.T) {
f, _ := ioutil.TempFile("", "test-no-read.txt")
defer os.Remove(f.Name())
oldReadFile := readFile
readFile = func(string) ([]byte, error) {
return nil, fmt.Errorf("read failed")
err := f.Chmod(0)
if err != nil {
t.Fatalf("failed to chmod file: %s", err)
}
_, err := New(f.Name(), noop, testErrCb(t))
_, err = New(f.Name(), noop, testErrCb(t))
if err == nil {
t.Fatalf("Expected New to return error when permission denied.")
readFile = oldReadFile
}
readFile = oldReadFile
}
func TestFirstError(t *testing.T) {
@ -184,11 +182,10 @@ func TestReloadFailure(t *testing.T) {
time.Sleep(15 * time.Millisecond)
// Create a file with no permissions
oldReadFile := readFile
readFile = func(string) ([]byte, error) {
return nil, fmt.Errorf("permisssion denied")
err = ioutil.WriteFile(filename, []byte("second body"), 0)
if err != nil {
t.Fatal(err)
}
fakeTick <- time.Now()
select {
case r := <-reloads:
@ -198,8 +195,11 @@ func TestReloadFailure(t *testing.T) {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for reload")
}
readFile = oldReadFile
err = os.Remove(filename)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filename, []byte("third body"), 0644)
if err != nil {
t.Fatal(err)

21
test.sh
View File

@ -72,6 +72,15 @@ function die() {
exit 1
}
function build_certbot() {
run git clone \
https://www.github.com/certbot/certbot.git \
$CERTBOT_PATH || exit 1
cd $CERTBOT_PATH
run ./tools/venv.sh
cd -
}
function run_unit_tests() {
if [ "${TRAVIS}" == "true" ]; then
@ -179,20 +188,18 @@ if [[ "$RUN" =~ "integration" ]] ; then
start_context "integration"
if [ -z "$CERTBOT_PATH" ]; then
export CERTBOT_PATH=$(mktemp -d -t cbpXXXX)
export CERTBOT_PATH=$(mktemp -d -t leXXXX)
echo "------------------------------------------------"
echo "--- Checking out letsencrypt client is slow. ---"
echo "--- Recommend setting \$CERTBOT_PATH to ---"
echo "--- client repo with initialized virtualenv ---"
echo "------------------------------------------------"
run git clone \
https://www.github.com/certbot/certbot.git \
$CERTBOT_PATH || exit 1
build_certbot
elif [ ! -d "${CERTBOT_PATH}" ]; then
build_certbot
fi
if ! type certbot >/dev/null 2>/dev/null; then
source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate
fi
source ${CERTBOT_PATH}/venv/bin/activate
python test/integration-test.py --all
if [ "$?" != 0 ]; then

4
test/boulder-tools/Dockerfile
View File

@ -1,4 +0,0 @@
FROM golang:1.5
ADD build.sh /tmp/build.sh
RUN bash /tmp/build.sh

46
test/boulder-tools/build.sh
View File

@ -1,46 +0,0 @@
#!/bin/bash -ex
# Boulder deps
apt-get update
apt-get install -y --no-install-recommends apt-transport-https ca-certificates
curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
cat >/etc/apt/sources.list.d/bouldertools.list <<EOAPT
deb https://deb.nodesource.com/node_4.x trusty main
deb-src https://deb.nodesource.com/node_4.x trusty main
deb http://ftp.debian.org/debian jessie-backports main
EOAPT
apt-get update
apt-get install -y --no-install-recommends -t jessie-backports letsencrypt python-letsencrypt-apache
apt-get install -y --no-install-recommends \
libltdl-dev \
mariadb-client-core-10.0 \
nodejs \
rpm \
ruby \
ruby-dev \
rsyslog \
softhsm \
protobuf-compiler &
# Install port forwarder, database migration tool, and testing tools.
GOBIN=/usr/local/bin GOPATH=/tmp/gopath go get \
github.com/jsha/listenbuddy \
bitbucket.org/liamstask/goose/cmd/goose \
github.com/golang/lint/golint \
github.com/golang/mock/mockgen \
github.com/golang/protobuf/proto \
github.com/golang/protobuf/protoc-gen-go \
github.com/kisielk/errcheck \
github.com/mattn/goveralls \
github.com/modocache/gover \
github.com/tools/godep \
golang.org/x/tools/cmd/stringer \
golang.org/x/tools/cover &
wait
gem install fpm
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

4
test/certbot
View File

@ -1,4 +0,0 @@
#!/bin/bash
#
# Temporary shim until the letsencrypt Debian package ships `certbot`
exec letsencrypt "$@"

5
test/create_db.sh
View File

@ -19,6 +19,9 @@ fi
# to the format we use in production, MIXED.
mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';"
# Drop all users to get a fresh start
mysql $dbconn < test/drop_users.sql
for dbenv in $DBENVS; do
(
db="boulder_sa_${dbenv}"
@ -39,8 +42,6 @@ for dbenv in $DBENVS; do
if [[ ${MYSQL_CONTAINER} ]]; then
sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \
mysql $dbconn -D $db || die "unable to add users to ${db}"
elif mysqld -V | grep "10.0"; then
mysql $dbconn -D $db < test/mariadb100_users.sql
else
sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \
mysql $dbconn -D $db < $USERS_SQL || die "unable to add users to ${db}"

2
test/ct-test-srv/main.go
View File

@ -142,7 +142,7 @@ func main() {
is := integrationSrv{key: key}
s := &http.Server{
Addr: "0.0.0.0:4500",
Addr: "localhost:4500",
Handler: http.HandlerFunc(is.handler),
}
log.Fatal(s.ListenAndServe())

4
test/docker-environment
View File

@ -1,4 +0,0 @@
PATH=/go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
GOPATH=/go
GOBIN=/go/src/github.com/letsencrypt/boulder/bin
GO15VENDOREXPERIMENT=1

5
test/drop_users.sql
View File

@ -3,9 +3,6 @@
-- Note that dropping a non-existing user produces an error that aborts the
-- script, so we first grant a harmless privilege to each user to ensure it
-- exists.
USE mysql;
GRANT USAGE ON *.* TO 'policy'@'localhost';
DROP USER 'policy'@'localhost';
GRANT USAGE ON *.* TO 'sa'@'localhost';
@ -24,5 +21,3 @@ GRANT USAGE ON *.* TO 'cert_checker'@'localhost';
DROP USER 'cert_checker'@'localhost';
GRANT USAGE ON *.* TO 'backfiller'@'localhost';
DROP USER 'backfiller'@'localhost';
GRANT USAGE ON *.* TO 'test_setup'@'localhost';
DROP USER 'test_setup'@'localhost';

6
test/entrypoint.sh
View File

@ -28,14 +28,10 @@ wait_tcp_port boulder-rabbitmq 5672
MYSQL_CONTAINER=1 $DIR/create_db.sh
# Set up rabbitmq exchange
rabbitmq-setup -server amqp://boulder-rabbitmq
go run cmd/rabbitmq-setup/main.go -server amqp://boulder-rabbitmq
if [[ $# -eq 0 ]]; then
exec ./start.py
fi
# TODO(jsha): Change to an unprivileged user before running commands. Currently,
# running as an unprivileged user causes the certbot integration test to fail
# during the test of the manual plugin. There's a call to killpg in there that
# kills the whole test, but only when run under `su buser -c "..."`
exec $@

6
test/mail-test-srv/main.go
View File

@ -16,7 +16,7 @@ import (
blog "github.com/letsencrypt/boulder/log"
)
var listenAPI = flag.String("http", "0.0.0.0:9381", "http port to listen on")
var apiPort = flag.String("http", "9381", "http port to listen on")
type rcvdMail struct {
From string
@ -162,7 +162,7 @@ func serveSMTP(l net.Listener) error {
}
func main() {
l, err := net.Listen("tcp", "0.0.0.0:9380")
l, err := net.Listen("tcp", ":9380")
if err != nil {
log.Fatalln("Couldn't bind for SMTP", err)
}
@ -170,7 +170,7 @@ func main() {
setupHTTP(http.DefaultServeMux)
go func() {
err := http.ListenAndServe(*listenAPI, http.DefaultServeMux)
err := http.ListenAndServe(":"+*apiPort, http.DefaultServeMux)
if err != nil {
log.Fatalln("Couldn't start HTTP server", err)
}

83
test/mariadb100_users.sql
View File

@ -1,83 +0,0 @@
--
-- Copyright 2015 ISRG. All rights reserved
-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
--
-- This file defines the default users for the primary database, used by
-- all the parts of Boulder except the Certificate Authority module, which
-- utilizes its own database.
--
-- Create users using MariaDB 10.0 syntax
-- Before setting up any privileges, we revoke existing ones to make sure we
-- start from a clean slate.
-- Note that dropping a non-existing user produces an error that aborts the
-- script, so we first grant a harmless privilege to each user to ensure it
-- exists.
GRANT USAGE ON *.* TO 'policy'@'localhost';
DROP USER 'policy'@'localhost';
GRANT USAGE ON *.* TO 'sa'@'localhost';
DROP USER 'sa'@'localhost';
GRANT USAGE ON *.* TO 'ocsp_resp'@'localhost';
DROP USER 'ocsp_resp'@'localhost';
GRANT USAGE ON *.* TO 'ocsp_update'@'localhost';
DROP USER 'ocsp_update'@'localhost';
GRANT USAGE ON *.* TO 'revoker'@'localhost';
DROP USER 'revoker'@'localhost';
GRANT USAGE ON *.* TO 'importer'@'localhost';
DROP USER 'importer'@'localhost';
GRANT USAGE ON *.* TO 'mailer'@'localhost';
DROP USER 'mailer'@'localhost';
GRANT USAGE ON *.* TO 'cert_checker'@'localhost';
DROP USER 'cert_checker'@'localhost';
GRANT USAGE ON *.* TO 'backfiller'@'localhost';
DROP USER 'backfiller'@'localhost';
GRANT USAGE ON *.* TO 'test_setup'@'localhost';
DROP USER 'test_setup'@'localhost';
-- Storage Authority
GRANT SELECT,INSERT,UPDATE ON authz TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE,DELETE ON pendingAuthorizations TO 'sa'@'localhost';
GRANT SELECT(id,Lockcol) ON pendingAuthorizations TO 'sa'@'localhost';
GRANT SELECT,INSERT ON certificates TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE ON certificateStatus TO 'sa'@'localhost';
GRANT SELECT,INSERT ON issuedNames TO 'sa'@'localhost';
GRANT SELECT,INSERT ON sctReceipts TO 'sa'@'localhost';
GRANT SELECT,INSERT ON deniedCSRs TO 'sa'@'localhost';
GRANT INSERT ON ocspResponses TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE ON registrations TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE ON challenges TO 'sa'@'localhost';
GRANT SELECT,INSERT on fqdnSets TO 'sa'@'localhost';
-- OCSP Responder
GRANT SELECT ON certificateStatus TO 'ocsp_resp'@'localhost';
GRANT SELECT ON ocspResponses TO 'ocsp_resp'@'localhost';
-- OCSP Generator Tool (Updater)
GRANT INSERT ON ocspResponses TO 'ocsp_update'@'localhost';
GRANT SELECT ON certificates TO 'ocsp_update'@'localhost';
GRANT SELECT,UPDATE ON certificateStatus TO 'ocsp_update'@'localhost';
GRANT SELECT ON sctReceipts TO 'ocsp_update'@'localhost';
-- Revoker Tool
GRANT SELECT ON registrations TO 'revoker'@'localhost';
GRANT SELECT ON certificates TO 'revoker'@'localhost';
GRANT SELECT,INSERT ON deniedCSRs TO 'revoker'@'localhost';
-- External Cert Importer
GRANT SELECT,INSERT,UPDATE,DELETE ON identifierData TO 'importer'@'localhost';
GRANT SELECT,INSERT,UPDATE,DELETE ON externalCerts TO 'importer'@'localhost';
-- Expiration mailer
GRANT SELECT ON certificates TO 'mailer'@'localhost';
GRANT SELECT,UPDATE ON certificateStatus TO 'mailer'@'localhost';
GRANT SELECT ON fqdnSets TO 'mailer'@'localhost';
-- Cert checker
GRANT SELECT ON certificates TO 'cert_checker'@'localhost';
-- Test setup and teardown
GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';

2
test/run-docker.sh
View File

@ -36,7 +36,7 @@ if [[ "$(is_running boulder-mysql)" != "true" ]]; then
docker run -d \
-e MYSQL_ALLOW_EMPTY_PASSWORD=yes \
--name boulder-mysql \
mariadb:10.1 mysqld --bind-address=0.0.0.0
mariadb:10.0 mysqld --bind-address=0.0.0.0
fi
if [[ "$(is_running boulder-rabbitmq)" != "true" ]]; then

16
test/sa_db_users.sql
View File

@ -14,18 +14,6 @@
-- drop command will fail. So we grant the dummy `USAGE` privilege to make sure
-- the user exists and then drop the user.
-- These lines require MariaDB 10.1
CREATE USER IF NOT EXISTS 'policy'@'localhost';
CREATE USER IF NOT EXISTS 'sa'@'localhost';
CREATE USER IF NOT EXISTS 'ocsp_resp'@'localhost';
CREATE USER IF NOT EXISTS 'revoker'@'localhost';
CREATE USER IF NOT EXISTS 'importer'@'localhost';
CREATE USER IF NOT EXISTS 'mailer'@'localhost';
CREATE USER IF NOT EXISTS 'cert_checker'@'localhost';
CREATE USER IF NOT EXISTS 'ocsp_update'@'localhost';
CREATE USER IF NOT EXISTS 'test_setup'@'localhost';
-- Storage Authority
GRANT SELECT,INSERT,UPDATE ON authz TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE,DELETE ON pendingAuthorizations TO 'sa'@'localhost';
@ -67,5 +55,9 @@ GRANT SELECT ON fqdnSets TO 'mailer'@'localhost';
-- Cert checker
GRANT SELECT ON certificates TO 'cert_checker'@'localhost';
-- Name set table backfiller
GRANT SELECT ON certificates to 'backfiller'@'localhost';
GRANT INSERT,SELECT ON fqdnSets to 'backfiller'@'localhost';
-- Test setup and teardown
GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';