diff --git a/.travis.yml b/.travis.yml index 8aebb2962..98c6d809f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,9 +65,6 @@ jobs: for f in $( grep -lR --include=Dockerfile\* go-deps: . ) ; do validate_go_deps_tag $f done - for f in $( grep -lR --include=Dockerfile\* proxy-deps: . ) ; do - validate_proxy_deps_tag $f - done ) # Push container images to Google Container Registry. @@ -123,10 +120,11 @@ jobs: - | export CONDUIT_TAG=$(. bin/_tag.sh ; clean_head_root_tag) echo "CONDUIT_TAG=${CONDUIT_TAG}" - - export PROXY_RELEASE=1 BUILD_DEBUG=1 DOCKER_TRACE=1 + - export BUILD_DEBUG=1 DOCKER_TRACE=1 script: - - bin/docker-build + # Tests are run in the `test` stage, se-running them here would be redundant/slow. #280 + - SKIP_TESTS=1 bin/docker-build after_success: - bin/docker-push-deps diff --git a/BUILD.md b/BUILD.md index ef28305db..a571b9b1d 100644 --- a/BUILD.md +++ b/BUILD.md @@ -327,6 +327,23 @@ To connect to a live `proxy-api` at `localhost:8086`: bin/go-run controller/cmd/proxy-api ``` +### Docker + +The `bin/docker-build-proxy` script builds the proxy: + +```bash +DOCKER_TRACE=1 PROXY_UNOPTIMIZED=1 PROXY_SKIP_TESTS=1 bin/docker-build-proxy +``` + +It supports two environment variables: + +- `PROXY_UNOPTIMIZED` -- When set and non-empty, produces unoptimized build artifacts, + which reduces build times at the expense of runtime performance. Changing this will + likely invalidate a substantial portion of Docker's cache. +- `PROXY_SKIP_TESTS` -- When set and non-empty, prevents the proxy's tests from being run + during the build. Changing this setting will not invalidate Docker's cache. + + ### Testing ```bash @@ -353,12 +370,7 @@ hard-coded SHA's: - [`Gopkg.lock`](Gopkg.lock) - [`Dockerfile-go-deps`](Dockerfile-go-deps) -`gcr.io/runconduit/proxy-deps` depends on -- [`Cargo.lock`](Cargo.lock) -- [`proxy/Dockerfile-deps`](proxy/Dockerfile-deps) - -The `bin/update-proxy-deps-shas` and `bin/update-go-deps-shas` must be run when their -respective dependencies change. +`bin/update-go-deps-shas` must be run when go dependencies change. # Build Architecture @@ -376,7 +388,6 @@ build_architecture "cli/Dockerfile" [color=lightblue, style=filled, shape=rect]; "cli/Dockerfile-bin" [color=lightblue, style=filled, shape=rect]; "proxy/Dockerfile" [color=lightblue, style=filled, shape=rect]; - "proxy/Dockerfile-deps" [color=lightblue, style=filled, shape=rect]; "proxy-init/Dockerfile" [color=lightblue, style=filled, shape=rect]; "proxy-init/integration-test/iptables/Dockerfile-tester" [color=lightblue, style=filled, shape=rect]; "web/Dockerfile" [color=lightblue, style=filled, shape=rect]; @@ -422,13 +433,8 @@ build_architecture "docker-build-proxy" -> "_docker.sh"; "docker-build-proxy" -> "_tag.sh"; "docker-build-proxy" -> "docker-build-base"; - "docker-build-proxy" -> "docker-build-go-deps"; "docker-build-proxy" -> "proxy/Dockerfile"; - "docker-build-proxy-deps" -> "_docker.sh"; - "docker-build-proxy-deps" -> "_tag.sh"; - "docker-build-proxy-deps" -> "proxy/Dockerfile-deps"; - "docker-build-proxy-init" -> "_docker.sh"; "docker-build-proxy-init" -> "_tag.sh"; "docker-build-proxy-init" -> "docker-build-base"; @@ -483,9 +489,6 @@ build_architecture "update-go-deps-shas" -> "controller/Dockerfile"; "update-go-deps-shas" -> "proxy-init/Dockerfile"; "update-go-deps-shas" -> "web/Dockerfile"; - - "update-proxy-deps-shas" -> "_tag.sh"; - "update-proxy-deps-shas" -> "proxy/Dockerfile"; } build_architecture diff --git a/bin/_tag.sh b/bin/_tag.sh index 2d85116cd..8b7cba12a 100644 --- a/bin/_tag.sh +++ b/bin/_tag.sh @@ -6,10 +6,6 @@ git_sha_head() { git rev-parse --short=8 HEAD } -proxy_deps_sha() { - cat Cargo.lock proxy/Dockerfile-deps | shasum - | awk '{print $1}' |cut -c 1-8 -} - go_deps_sha() { cat Gopkg.lock Dockerfile-go-deps | shasum - | awk '{print $1}' |cut -c 1-8 } @@ -66,14 +62,9 @@ validate_tag() { # These functions should be called by any docker-build-* script that relies on # Go or Rust dependencies. To confirm the set of scripts that should call this # function, run: -# $ grep -ER 'docker-build-(go|proxy)-deps' . +# $ grep -ER 'docker-build-go-deps' . validate_go_deps_tag() { file="$1" validate_tag "$file" "gcr.io/runconduit/go-deps" "$(go_deps_sha)" } - -validate_proxy_deps_tag() { - file="$1" - validate_tag "$file" "gcr.io/runconduit/proxy-deps" "$(proxy_deps_sha)" -} diff --git a/bin/docker-build b/bin/docker-build index ee80575ca..8e3b817b1 100755 --- a/bin/docker-build +++ b/bin/docker-build @@ -9,6 +9,7 @@ fi bin/docker-build-controller bin/docker-build-web -bin/docker-build-proxy bin/docker-build-proxy-init bin/docker-build-cli + +bin/docker-build-proxy diff --git a/bin/docker-build-proxy b/bin/docker-build-proxy index 3544d4916..51632b2ae 100755 --- a/bin/docker-build-proxy +++ b/bin/docker-build-proxy @@ -10,13 +10,6 @@ fi . bin/_docker.sh . bin/_tag.sh -dockerfile=proxy/Dockerfile - -validate_proxy_deps_tag $dockerfile - -( - bin/docker-build-base - bin/docker-build-proxy-deps -) >/dev/null - -docker_build proxy "$(head_root_tag)" $dockerfile --build-arg="RELEASE=${PROXY_RELEASE:-1}" +docker_build proxy "$(head_root_tag)" proxy/Dockerfile \ + --build-arg="PROXY_SKIP_TESTS=${PROXY_SKIP_TESTS:-}" \ + --build-arg="PROXY_UNOPTIMIZED=${PROXY_UNOPTIMIZED:-}" diff --git a/bin/docker-build-proxy-deps b/bin/docker-build-proxy-deps deleted file mode 100755 index 43585fb10..000000000 --- a/bin/docker-build-proxy-deps +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -# Builds (or pulls) our proxy-deps docker image. - -set -eu - -if [ $# -ne 0 ]; then - echo "no arguments allowed for $(basename $0), given: $@" >&2 - exit 64 -fi - -. bin/_docker.sh -. bin/_tag.sh - -tag=$(proxy_deps_sha) - -if (docker_pull proxy-deps "${tag}"); then - echo "$(docker_repo proxy-deps):${tag}" -else - docker_build proxy-deps "${tag}" proxy/Dockerfile-deps -fi diff --git a/bin/docker-images b/bin/docker-images index e4a202825..a3435ff9f 100755 --- a/bin/docker-images +++ b/bin/docker-images @@ -22,4 +22,3 @@ docker_image cli "${tag}" docker_image cli-bin "${tag}" docker_image go-deps "$(go_deps_sha)" -docker_image proxy-deps "$(proxy_deps_sha)" diff --git a/bin/docker-pull-deps b/bin/docker-pull-deps index cdd652b89..a39d3fb63 100755 --- a/bin/docker-pull-deps +++ b/bin/docker-pull-deps @@ -7,4 +7,3 @@ set -eu docker_pull base 2017-10-30.01 || true docker_pull go-deps "$(go_deps_sha)" || true -docker_pull proxy-deps "$(proxy_deps_sha)" || true diff --git a/bin/docker-push-deps b/bin/docker-push-deps index a2ec2f045..068d6d5d6 100755 --- a/bin/docker-push-deps +++ b/bin/docker-push-deps @@ -7,4 +7,3 @@ set -eu docker_push base 2017-10-30.01 docker_push go-deps "$(go_deps_sha)" -docker_push proxy-deps "$(proxy_deps_sha)" diff --git a/bin/update-proxy-deps-shas b/bin/update-proxy-deps-shas deleted file mode 100755 index 9defafb3b..000000000 --- a/bin/update-proxy-deps-shas +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -set -eu - -# Updates the tag for `runconduit/proxy-deps` across all Dockerfiles in this repository. - -sha=$(. bin/_tag.sh ; proxy_deps_sha) - -for f in $( grep -lR --include=Dockerfile\* proxy-deps: . ) ; do - sed -E -i.bak -e "s|runconduit/proxy-deps:[^ ]+|runconduit/proxy-deps:${sha}|" "$f" - rm "$f".bak -done diff --git a/proxy/Dockerfile b/proxy/Dockerfile index 4b627e4f2..dc05b44e4 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,24 +1,74 @@ # Proxy build and runtime # -# Builds a slim runtime image with the conduit-proxy binary. - -## Build the rust proxy into a binary. +# When PROXY_UNOPTIMIZED is set and not empty, unoptimized rust artifacts are produced. +# This reduces build time and produces binaries with debug symbols, at the expense of +# runtime perforamnce. # -# If the RELEASE arg is set and non-empty, a release artifact is built. -FROM gcr.io/runconduit/proxy-deps:673c53de as build +# When PROXY_SKIP_TESTS is set and not empty, tests are not run. Otherwise, tests are run +# against either unoptimized or optimized proxy code, according to PROXY_UNOPTIMIZED. + +ARG RUST_IMAGE=rust:1.23.0 +ARG RUNTIME_IMAGE=gcr.io/runconduit/base:2017-10-30.01 + +## Builds the proxy as incrementally as possible. +FROM $RUST_IMAGE as build + WORKDIR /usr/src/conduit -# Ranked roughly from least to most likely to change. Cargo.lock is the least likely -# because it is supposed to be cached in the deps base image. -COPY proto ./proto -COPY proxy ./proxy -ARG RELEASE -RUN if [ -z "$RELEASE" ]; \ - then cargo build --frozen -p conduit-proxy && mv target/debug/conduit-proxy target/conduit-proxy ; \ - else cargo build --frozen -p conduit-proxy --release && mv target/release/conduit-proxy target/conduit-proxy ; \ + +# Fetch external dependencies. +# +# Mock out all local code and fetch external dependencies to ensure that external sources +# are cached. +RUN for d in proxy proxy/controller-grpc proxy/convert proxy/futures-mpsc-lossy proxy/router ; \ + do mkdir -p "${d}/src" && touch "${d}/src/lib.rs" ; \ + done +COPY Cargo.toml Cargo.lock ./ +COPY proxy/Cargo.toml proxy/Cargo.toml +COPY proxy/controller-grpc/Cargo.toml proxy/controller-grpc/Cargo.toml +COPY proxy/convert/Cargo.toml proxy/convert/Cargo.toml +COPY proxy/futures-mpsc-lossy/Cargo.toml proxy/futures-mpsc-lossy/Cargo.toml +COPY proxy/router/Cargo.toml proxy/router/Cargo.toml +RUN cargo fetch --locked + +# Build libraries, leaving the proxy and gRPC bindings mocked out. +COPY proxy/convert proxy/convert +COPY proxy/futures-mpsc-lossy proxy/futures-mpsc-lossy +COPY proxy/router proxy/router +ARG PROXY_UNOPTIMIZED +RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \ + then cargo build --frozen ; \ + else cargo build --frozen --release ; \ + fi + +# Build gRPC bindings, leaving the proxy mocked out. +COPY proto proto +COPY proxy/controller-grpc proxy/controller-grpc +RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \ + then cargo build -p conduit-proxy-controller-grpc --features=arbitrary --frozen ; \ + else cargo build -p conduit-proxy-controller-grpc --features=arbitrary --frozen --release ; \ + fi + +# Build the proxy binary using pre-built dependencies. +COPY proxy/src proxy/src +COPY proxy/tests proxy/tests +RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \ + then cargo build -p conduit-proxy --bin conduit-proxy --frozen ; \ + else cargo build -p conduit-proxy --bin conduit-proxy --frozen --release ; \ + fi +ARG PROXY_SKIP_TESTS +RUN if [ -n "$PROXY_SKIP_TESTS" ]; \ + then echo "tests skipped" ; \ + elif [ -n "$PROXY_UNOPTIMIZED" ]; \ + then cargo test -p conduit-proxy --frozen ; \ + else cargo test -p conduit-proxy --frozen --release ; \ + fi +RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \ + then mv target/debug/conduit-proxy target/conduit-proxy ; \ + else mv target/release/conduit-proxy target/conduit-proxy ; \ fi ## Install the proxy binary into the base runtime image. -FROM gcr.io/runconduit/base:2017-10-30.01 +FROM $RUNTIME_IMAGE as runtime COPY --from=build /usr/src/conduit/target/conduit-proxy /usr/local/bin/conduit-proxy -ENV CONDUIT_PROXY_LOG=info +ENV CONDUIT_PROXY_LOG=warn,conduit_proxy=info ENTRYPOINT ["/usr/local/bin/conduit-proxy"] diff --git a/proxy/Dockerfile-deps b/proxy/Dockerfile-deps deleted file mode 100644 index 299f812cd..000000000 --- a/proxy/Dockerfile-deps +++ /dev/null @@ -1,24 +0,0 @@ -# Proxy dependencies -# -# Fetches all required rust dependencies and caches library artifacts. All Conduit sources -# are omitted from the resulting image so that artifacts may be built from source over -# this image. -# -# When this file is changed, you must run `bin/update-proxy-deps-shas`. - -# Compile the application to ensure we've obtained all build dependencies and that they -# compile. -FROM rust:1.23.0 as build -WORKDIR /usr/src/conduit -COPY Cargo.toml Cargo.lock ./ -COPY proto ./proto -COPY proxy ./proxy -RUN cargo fetch --locked - -# Preserve dependency sources and build artifacts without maintaining conduit -# sources/artifacts. -FROM rust:1.23.0 -WORKDIR /usr/src/conduit -COPY --from=build $CARGO_HOME $CARGO_HOME -COPY --from=build /usr/src/conduit/Cargo.toml Cargo.toml -COPY --from=build /usr/src/conduit/Cargo.lock Cargo.lock