From 56160e0058d91f96244defec79bae29e5ad1b49e Mon Sep 17 00:00:00 2001 From: Olivier Gambier Date: Wed, 7 Oct 2015 13:38:55 -0700 Subject: [PATCH] Prepare build for plugins PR Signed-off-by: Olivier Gambier --- CONTRIBUTING.md | 42 ++++++++++++++++-------------------- Dockerfile | 3 +-- Makefile | 9 ++++++-- Makefile.inc | 12 +++-------- circle.yml | 3 +-- main.go => cmd/machine.go | 0 doc.go | 4 ++++ mk/build.mk | 40 +++++++++++++--------------------- mk/coverage.mk | 4 ++-- mk/main.mk | 12 ++++++++--- mk/release.mk | 12 ++++++----- mk/test.mk | 6 +++--- mk/validate.mk | 11 +++++++--- test/integration/run-bats.sh | 33 +++------------------------- 14 files changed, 81 insertions(+), 110 deletions(-) rename main.go => cmd/machine.go (100%) create mode 100644 doc.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c0124b44e..844268e75a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,21 +18,25 @@ guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md). The requirements to build Machine are: -1. A running instance of Docker (or alternatively a golang 1.5 development environment) +1. A running instance of Docker 2. The `bash` shell 3. [Make](https://www.gnu.org/software/make/) Call `export USE_CONTAINER=true` to instruct the build system to use containers to build. -If you want to build natively using golang instead, don't set this variable. + +# Alternative: build using go only + +Alternatively, you can build without docker, using only golang. + +[Install and setup go](https://golang.org/doc/install), then clone the machine repository inside your gopath. ## Building To build the docker-machine binary, simply run: - $ make + $ make build -From the Machine repository's root. You will now find a `bin/docker-machine` -binary at the root of the project. +From the Machine repository's root. You will now have a `bin/docker-machine`. You may call: @@ -61,6 +65,8 @@ To generate an html code coverage report of the Machine codebase, run: And navigate to http://localhost:8000 (hit `CTRL+C` to stop the server). +### Native build + Alternatively, if you are building natively, you can simply run: make coverage-html @@ -80,9 +86,9 @@ This will generate and open the report file: ### Build targets -Build a single, native machine binary: +Just build the machine binary itself: - make build-simple + make `pwd`/bin/docker-machine Build for all supported oses and architectures (binaries will be in the `bin` project subfolder): @@ -97,8 +103,7 @@ You can further control build options through the following environment variable DEBUG=true # enable debug build STATIC=true # build static (note: when cross-compiling, the build is always static) VERBOSE=true # verbose output - PARALLEL=X # lets you control build parallelism when cross-compiling multiple builds - PREFIX=folder + PREFIX=folder # put binaries in another folder (not the default `./bin`) Scrub build results: @@ -134,16 +139,12 @@ first make sure to [install it](https://github.com/sstephenson/bats#installing-b ### Basic Usage -Integration tests can be invoked calling `make test-integration`. +You first need to build, calling `make build`. -:warn: you cannot run integration test inside a container for now. -Be sure to unset the `USE_CONTAINER` env variable if you set it earlier, or alternatively -call directly `./test/integration/run-bats.sh` instead of `make test-integration`. +You can then invoke integration tests calling `DRIVER=foo make test-integration TESTSUITE`, where `TESTSUITE` is +one of the `test/integration` subfolder, and `foo` is the specific driver you want to test. -You can invoke a test or subset of tests for a particular driver. -To set the driver, use the `DRIVER` environment variable. - -To invoke just one test: +Examples: ```console $ DRIVER=virtualbox make test-integration test/integration/core/core-commands.bats @@ -170,13 +171,6 @@ Cleaning up machines... Successfully removed bats-virtualbox-test ``` -To invoke a shared test with a different driver: - -```console -$ DRIVER=digitalocean make test-integration test/integration/core/core-commands.bats -... -``` - To invoke a directory of tests recursively: ```console diff --git a/Dockerfile b/Dockerfile index 9859935c92..bcbb450205 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM golang:1.5 -RUN go get github.com/mitchellh/gox \ - github.com/golang/lint/golint \ +RUN go get github.com/golang/lint/golint \ github.com/mattn/goveralls \ golang.org/x/tools/cover \ github.com/tools/godep \ diff --git a/Makefile b/Makefile index 51812be8d8..4998d01343 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ -# Plain make targets if not requested inside a container -ifeq ($(USE_CONTAINER),) +# # Plain make targets if not requested inside a container +ifneq (,$(findstring test-integration,$(MAKECMDGOALS))) + include Makefile.inc + include mk/main.mk +else ifeq ($(USE_CONTAINER),) include Makefile.inc include mk/main.mk else @@ -7,6 +10,8 @@ else DOCKER_IMAGE_NAME := "docker-machine-build" DOCKER_CONTAINER_NAME := "docker-machine-build-container" +test: FORCE + %: @docker build -t $(DOCKER_IMAGE_NAME) . diff --git a/Makefile.inc b/Makefile.inc index 3e92741e1e..5c52d7e2bb 100644 --- a/Makefile.inc +++ b/Makefile.inc @@ -11,7 +11,7 @@ STATIC ?= VERBOSE ?= # Build tags BUILDTAGS ?= -# Adjust parallelism for gox +# Adjust number of parallel builds (XXX not used) PARALLEL ?= -1 # Coverage default directory COVERAGE_DIR ?= cover @@ -20,20 +20,14 @@ USE_CONTAINER ?= # List of cross compilation targets ifeq ($(TARGET_OS),) - TARGET_OS := darwin freebsd linux windows + TARGET_OS := darwin linux windows endif ifeq ($(TARGET_ARCH),) - TARGET_ARCH := amd64 arm 386 + TARGET_ARCH := amd64 386 endif # Output prefix, defaults to local directory if not specified ifeq ($(PREFIX),) PREFIX := $(shell pwd) endif - -default: build -clean: coverage-clean build-clean -build: build-simple -test: dco fmt vet test-short -validate: dco fmt vet lint test-short test-long \ No newline at end of file diff --git a/circle.yml b/circle.yml index ab3ed50580..f9f1f8c383 100644 --- a/circle.yml +++ b/circle.yml @@ -9,7 +9,6 @@ machine: post: - gvm install go1.5 -B --name=stable - - gvm use stable && go get github.com/mitchellh/gox environment: # Convenient shortcuts to "common" locations @@ -30,7 +29,7 @@ dependencies: test: pre: - gvm use stable && go version - - gvm use stable && TARGET_ARCH=amd64 TARGET_OS=linux make build-x: + - gvm use stable && make build-simple: pwd: $BASE_STABLE override: diff --git a/main.go b/cmd/machine.go similarity index 100% rename from main.go rename to cmd/machine.go diff --git a/doc.go b/doc.go new file mode 100644 index 0000000000..3091fb9ea5 --- /dev/null +++ b/doc.go @@ -0,0 +1,4 @@ +// Machine defines interfaces to manage a variety of docker instances +// deployed on different backends (VMs, baremetal). +// The goal is to allow users get from zero to docker as fast as possible. +package machine diff --git a/mk/build.mk b/mk/build.mk index 9f0674f574..129343fee9 100644 --- a/mk/build.mk +++ b/mk/build.mk @@ -1,33 +1,23 @@ build-clean: @rm -f $(PREFIX)/bin/* -# Simple build -build-simple: $(PREFIX)/bin/$(PKG_NAME) +# Cross builder helper +define gocross + GOOS=$(1) GOARCH=$(2) CGO_ENABLED=0 go build -o $(PREFIX)/bin/$(1)-$(2)/docker-$(patsubst cmd/%.go,%,$3) \ + -a $(VERBOSE_GO) -tags "static_build netgo $(BUILDTAGS)" -installsuffix netgo -ldflags "$(GO_LDFLAGS) \ + -extldflags -static" $(GO_GCFLAGS) $(3); +endef # XXX building with -a fails in debug (with -N -l) ???? -$(PREFIX)/bin/$(PKG_NAME): $(shell find . -type f -name '*.go') - @go build -o $@ $(VERBOSE_GO) -tags "$(BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" $(GO_GCFLAGS) ./main.go +$(PREFIX)/bin/docker-%: ./cmd/%.go $(shell find . -type f -name '*.go') + $(GO) build -o $@ $(VERBOSE_GO) -tags "$(BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" $(GO_GCFLAGS) $< -# Cross-build: careful, does always rebuild! -build-x: clean - $(if $(GOX), , \ - $(error Please install gox: go get -u github.com/mitchellh/gox)) - @$(GOX) \ - -os "$(TARGET_OS)" \ - -arch "$(TARGET_ARCH)" \ - -output="$(PREFIX)/bin/docker-machine_{{.OS}}-{{.Arch}}" \ - -ldflags="$(GO_LDFLAGS)" \ - -tags="$(BUILDTAGS)" \ - -gcflags="$(GO_GCFLAGS)" \ - -parallel=$(PARALLEL) \ - -rebuild $(VERBOSE_GOX) +# Native build +build-simple: $(patsubst ./cmd/%.go,$(PREFIX)/bin/docker-%,$(wildcard ./cmd/*.go)) -# Cross builder helper -# define gocross -# GOOS=$(1) GOARCH=$(2) CGO_ENABLED=0 go build -o $(PREFIX)/$(PKG_NAME)_$(1)-$(2) \ -# -a $(VERBOSE_GO) -tags "static_build netgo $(BUILDTAGS)" -installsuffix netgo -ldflags "$(GO_LDFLAGS) -extldflags -static" $(GO_GCFLAGS) ./main.go; -# endef +# Cross compilation targets +build-x-%: ./cmd/%.go $(shell find . -type f -name '*.go') + @$(foreach GOARCH,$(TARGET_ARCH),$(foreach GOOS,$(TARGET_OS),$(call gocross,$(GOOS),$(GOARCH),$<))) -# Native build-x (no gox) -# build-x: $(shell find . -type f -name '*.go') -# @$(foreach GOARCH,$(TARGET_ARCH),$(foreach GOOS,$(TARGET_OS),$(call gocross,$(GS),$(GA)))) \ No newline at end of file +# Cross-build +build-x: $(patsubst ./cmd/%.go,build-x-%,$(wildcard ./cmd/*.go)) \ No newline at end of file diff --git a/mk/coverage.mk b/mk/coverage.mk index 368f10c55e..751b4375f6 100644 --- a/mk/coverage.mk +++ b/mk/coverage.mk @@ -37,9 +37,9 @@ coverage-clean: $(COVERAGE_PROFILE): $(shell find . -type f -name '*.go') @mkdir -p "$(COVERAGE_OUTPUT)/coverage" - @$(foreach PKG,$(PKGS), go test $(VERBOSE) -tags "$(BUILDTAGS)" -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_OUTPUT)/coverage/`echo $(PKG) | tr "/" "-"`.cover" "$(PKG)";) + @$(foreach PKG,$(PKGS), go test $(VERBOSE_GO) -tags "$(BUILDTAGS)" -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_OUTPUT)/coverage/`echo $(PKG) | tr "/" "-"`.cover" "$(PKG)";) @echo "mode: $(COVERAGE_MODE)" > "$(COVERAGE_PROFILE)" @grep -h -v "^mode:" "$(COVERAGE_OUTPUT)/coverage"/*.cover >> "$(COVERAGE_PROFILE)" $(COVERAGE_HTML): $(COVERAGE_PROFILE) - @go tool cover -html="$(COVERAGE_PROFILE)" -o "$(COVERAGE_HTML)" + $(GO) tool cover -html="$(COVERAGE_PROFILE)" -o "$(COVERAGE_HTML)" diff --git a/mk/main.mk b/mk/main.mk index 9a67fad0c9..98a2dda30e 100644 --- a/mk/main.mk +++ b/mk/main.mk @@ -32,10 +32,10 @@ endif # Honor verbose VERBOSE_GO := -VERBOSE_GOX := +GO := @go ifeq ($(VERBOSE),true) VERBOSE_GO := -v - VERBOSE_GOX := -verbose + GO := go endif include mk/build.mk @@ -50,4 +50,10 @@ include mk/validate.mk .all_test: test-short test-long test-integration .all_validate: dco fmt vet lint -.PHONY: .all_build .all_coverage .all_release .all_test .all_validate +default: build +clean: coverage-clean build-clean +build: build-simple +test: dco fmt vet test-short +validate: dco fmt vet lint test-short test-long + +.PHONY: .all_build .all_coverage .all_release .all_test .all_validate test build validate clean diff --git a/mk/release.mk b/mk/release.mk index 66710334a4..2f439391f9 100644 --- a/mk/release.mk +++ b/mk/release.mk @@ -1,12 +1,14 @@ -# XXX FIXME: suboptimal, as it will sign signatures files if they are there release-checksum: - $(foreach MACHINE_FILE, $(wildcard $(PREFIX)/bin/*), \ + $(foreach MACHINE_FILE, $(wildcard $(PREFIX)/bin/*.zip), \ $(shell openssl dgst -sha256 < "$(MACHINE_FILE)" > "$(MACHINE_FILE).sha256" && \ openssl dgst -md5 < "$(MACHINE_FILE)" > "$(MACHINE_FILE).md5" \ )) @: -release: clean dco fmt test test-long build-x release-checksum +release-pack: + find ./bin -type d -mindepth 1 -exec zip -r -j {}.zip {} \; + +release: clean dco fmt test test-long build-x release-pack release-checksum $(if $(GITHUB_TOKEN), , \ $(error GITHUB_TOKEN must be set for github-release)) @@ -28,7 +30,7 @@ release: clean dco fmt test test-long build-x release-checksum --description "" \ --pre-release - $(foreach MACHINE_FILE, $(wildcard $(PREFIX)/bin/*), \ + $(foreach MACHINE_FILE, $(wildcard $(PREFIX)/bin/*.zip), \ $(shell github-release upload \ --user $(GH_USER) \ --repo $(GH_REPO) \ @@ -39,4 +41,4 @@ release: clean dco fmt test test-long build-x release-checksum ) %: - @: + @: \ No newline at end of file diff --git a/mk/test.mk b/mk/test.mk index cf3b9a5bc4..62a0d8b237 100644 --- a/mk/test.mk +++ b/mk/test.mk @@ -1,12 +1,12 @@ # Quick test. You can bypass long tests using: `if testing.Short() { t.Skip("Skipping in short mode.") }` test-short: - @go test $(VERBOSE_GO) -test.short -tags "$(BUILDTAGS)" $(PKGS) + $(GO) test $(VERBOSE_GO) -test.short -tags "$(BUILDTAGS)" $(PKGS) # Runs long tests also, plus race detection test-long: - @go test $(VERBOSE_GO) -race -tags "$(BUILDTAGS)" $(PKGS) + $(GO) test $(VERBOSE_GO) -race -tags "$(BUILDTAGS)" $(PKGS) -test-integration: build +test-integration: $(eval TESTSUITE=$(filter-out $@,$(MAKECMDGOALS))) test/integration/run-bats.sh $(TESTSUITE) diff --git a/mk/validate.mk b/mk/validate.mk index c953dd4061..be8fdfef01 100644 --- a/mk/validate.mk +++ b/mk/validate.mk @@ -1,6 +1,12 @@ -# Validate DCO on all history +# Validate DCO on all history +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path)))) + +# XXX vendorized script miss exec bit, hence the gymnastic +# plus the path resolution... +# TODO migrate away from the shell script and have a make equivalent instead dco: - @script/validate-dco + @echo `bash $(current_dir)/../script/validate-dco` # Fmt fmt: @@ -15,4 +21,3 @@ lint: $(if $(GOLINT), , \ $(error Please install golint: go get -u github.com/golang/lint/golint)) @test -z "$$($(GOLINT) ./... 2>&1 | grep -v vendor/ | grep -v Godeps/ | tee /dev/stderr)" - diff --git a/test/integration/run-bats.sh b/test/integration/run-bats.sh index ccadf940b2..884405641a 100755 --- a/test/integration/run-bats.sh +++ b/test/integration/run-bats.sh @@ -5,12 +5,6 @@ set -e # Wrapper script to run bats tests for various drivers. # Usage: DRIVER=[driver] ./run-bats.sh [subtest] -function build_machine() { - cd ${MACHINE_ROOT} - make build-x - cd - -} - function quiet_run () { if [[ "$VERBOSE" == "1" ]]; then "$@" @@ -27,6 +21,7 @@ function cleanup_machines() { } function machine() { + export PATH="$MACHINE_ROOT"/bin:$PATH "$MACHINE_ROOT"/bin/"$MACHINE_BIN_NAME" "$@" } @@ -46,32 +41,10 @@ function run_bats() { done } -# Platform and architecture information is used to correctly identify the -# binary we want to test. -PLATFORM=`uname -s | tr '[:upper:]' '[:lower:]'` -case "$(uname -m)" in - arm*) - ARCH="arm" - ;; - x86_64) - ARCH="amd64" - ;; - i*86) - ARCH="386" - ;; - *) - ARCH="$(uname -m)" -esac - # Set this ourselves in case bats call fails EXIT_STATUS=0 export BATS_FILE="$1" -# build machine binary if needed -if [ ! -e "$MACHINE_ROOT"/bin/"$MACHINE_BIN_NAME" ]; then - build_machine -fi - if [[ -z "$DRIVER" ]]; then echo "You must specify the DRIVER environment variable." exit 1 @@ -92,7 +65,7 @@ export BASE_TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) export MACHINE_ROOT="$BASE_TEST_DIR/../.." export NAME="bats-$DRIVER-test" export MACHINE_STORAGE_PATH="/tmp/machine-bats-test-$DRIVER" -export MACHINE_BIN_NAME=docker-machine_$PLATFORM-$ARCH +export MACHINE_BIN_NAME=docker-machine export BATS_LOG="$MACHINE_ROOT/bats.log" # This function gets used in the integration tests, so export it. @@ -107,4 +80,4 @@ if [[ -d "$MACHINE_STORAGE_PATH" ]]; then rm -r "$MACHINE_STORAGE_PATH" fi -exit ${EXIT_STATUS} +exit ${EXIT_STATUS} \ No newline at end of file