From 91201678c6f58a5e46fe302bed9eb55a7ddcf693 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 4 Jan 2016 21:50:15 -0800 Subject: [PATCH] Refactor multi-arch support (especially for new mostly-official multi-arch official images) See https://github.com/docker-library/official-images/blob/99433d2ca217bd9479169c93a20b6f6530f94856/README.md#architectures-other-than-amd64 for some context of where these images come from. Signed-off-by: Andrew "Tianon" Page --- Dockerfile | 51 ++++++------ Dockerfile.armhf | 91 ++++++++++---------- Dockerfile.ppc64le | 123 +++++++++++++++++++++++++--- Dockerfile.s390x | 120 ++++++++++++++++++++++++--- Makefile | 27 ++++-- hack/make/.detect-daemon-osarch | 18 ++++ hack/make/.ensure-frozen-images | 89 ++++++++------------ hack/make/.integration-daemon-setup | 9 +- 8 files changed, 369 insertions(+), 159 deletions(-) create mode 100644 hack/make/.detect-daemon-osarch diff --git a/Dockerfile b/Dockerfile index ff46194188..4caa32e4a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,16 +23,15 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM ubuntu:14.04 -MAINTAINER Tianon Gravi (@tianon) +FROM ubuntu:trusty # add zfs ppa -RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 -RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list +RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 +RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list # add llvm repo -RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 15CF4D18AF4F7421 -RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list +RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 +RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list # Packaged dependencies RUN apt-get update && apt-get install -y \ @@ -72,19 +71,25 @@ RUN apt-get update && apt-get install -y \ && ln -snf /usr/bin/clang++-3.8 /usr/local/bin/clang++ # Get lvm2 source for compiling statically -RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 +ENV LVM2_VERSION 2.02.103 +RUN mkdir -p /usr/local/lvm2 \ + && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ + | tar -xzC /usr/local/lvm2 --strip-components=1 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags # Compile and install lvm2 RUN cd /usr/local/lvm2 \ - && ./configure --enable-static_link \ + && ./configure \ + --build="$(gcc -print-multiarch)" \ + --enable-static_link \ && make device-mapper \ && make install_device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go ENV GO_VERSION 1.5.2 -RUN curl -sSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" | tar -v -C /usr/local -xz +RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" \ + | tar -xzC /usr/local ENV PATH /go/bin:/usr/local/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor @@ -128,17 +133,17 @@ ENV PATH /osxcross/target/bin:$PATH # install seccomp # this can be changed to the ubuntu package libseccomp-dev if dockerinit is removed, # we need libseccomp.a (which the package does not provide) for dockerinit -ENV SECCOMP_VERSION v2.2.3 +ENV SECCOMP_VERSION 2.2.3 RUN set -x \ - && export SECCOMP_PATH=$(mktemp -d) \ - && git clone https://github.com/seccomp/libseccomp.git "$SECCOMP_PATH" \ + && export SECCOMP_PATH="$(mktemp -d)" \ + && curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \ + | tar -xzC "$SECCOMP_PATH" --strip-components=1 \ && ( \ cd "$SECCOMP_PATH" \ - && git checkout "$SECCOMP_VERSION" \ - && ./autogen.sh \ - && ./configure --prefix=/usr \ + && ./configure --prefix=/usr/local \ && make \ && make install \ + && ldconfig \ ) \ && rm -rf "$SECCOMP_PATH" @@ -196,7 +201,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - busybox:latest@sha256:eb3c0d4680f9213ee5f348ea6d39489a1f85a318a2ae09e012c426f78252a6d2 \ + busybox:latest@sha256:e4f93f6ed15a0cdd342f5aae387886fba0ab98af0a102da6276eaf24d6e6ade0 \ debian:jessie@sha256:24a900d1671b269d6640b4224e7b63801880d8e3cb2bcbfaa10a5dddcf4469ed \ hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7 # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is) @@ -204,8 +209,8 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ # Download man page generator RUN set -x \ && export GOPATH="$(mktemp -d)" \ - && git clone -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ - && git clone -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ + && git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ + && git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ && go get -v -d github.com/cpuguy83/go-md2man \ && go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \ && rm -rf "$GOPATH" @@ -220,12 +225,12 @@ RUN set -x \ && rm -rf "$GOPATH" # Build/install the tool for embedding resources in Windows binaries -ENV RSRC_COMMIT e48dbf1b7fc464a9e85fcec450dddf80816b76e0 +ENV RSRC_VERSION v2 RUN set -x \ - && git clone https://github.com/akavel/rsrc.git /go/src/github.com/akavel/rsrc \ - && cd /go/src/github.com/akavel/rsrc \ - && git checkout -q $RSRC_COMMIT \ - && go install -v + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \ + && go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \ + && rm -rf "$GOPATH" # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 95a7201d5d..19188287ec 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -23,8 +23,7 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM ioft/armhf-ubuntu:14.04 -MAINTAINER Govinda Fichtner (@_beagile_) +FROM armhf/ubuntu:trusty # Packaged dependencies RUN apt-get update && apt-get install -y \ @@ -40,6 +39,7 @@ RUN apt-get update && apt-get install -y \ git \ iptables \ jq \ + net-tools \ libapparmor-dev \ libcap-dev \ libltdl-dev \ @@ -52,38 +52,47 @@ RUN apt-get update && apt-get install -y \ python-mock \ python-pip \ python-websocket \ - s3cmd=1.1.0* \ xfsprogs \ tar \ --no-install-recommends # Get lvm2 source for compiling statically -RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 +ENV LVM2_VERSION 2.02.103 +RUN mkdir -p /usr/local/lvm2 \ + && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ + | tar -xzC /usr/local/lvm2 --strip-components=1 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags # Compile and install lvm2 RUN cd /usr/local/lvm2 \ - && ./configure --enable-static_link \ + && ./configure \ + --build="$(gcc -print-multiarch)" \ + --enable-static_link \ && make device-mapper \ && make install_device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go -ENV GO_VERSION 1.4.3 +#ENV GO_VERSION 1.5.2 # TODO update GO_TOOLS_COMMIT below when this updates to 1.5+ -RUN curl -fsSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz +ENV GO_VERSION 1.4.3 +RUN curl -fsSL "https://github.com/hypriot/golang-armbuilds/releases/download/v${GO_VERSION}/go${GO_VERSION}.linux-armv7.tar.gz" \ + | tar -xzC /usr/local +# temporarily using Hypriot's tarballs while we wait for official 1.6+ +#RUN curl -fsSL https://golang.org/dl/go${GO_VERSION}.linux-arm6.tar.gz \ +# | tar -xzC /usr/local ENV PATH /go/bin:/usr/local/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor # we're building for armhf, which is ARMv7, so let's be explicit about that +ENV GOARCH arm ENV GOARM 7 -RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1 - # This has been commented out and kept as reference because we don't support compiling with older Go anymore. # ENV GOFMT_VERSION 1.3.3 # RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt +#ENV GO_TOOLS_COMMIT 823804e1ae08dbb14eb807afc7db9993bc9e3cc3 # TODO update this sha when we upgrade to Go 1.5+ ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9 # Grab Go's cover tool for dead-simple code coverage testing @@ -94,6 +103,8 @@ RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \ && go install -v golang.org/x/tools/cmd/cover \ && go install -v golang.org/x/tools/cmd/vet # Grab Go's lint tool +#ENV GO_LINT_COMMIT 32a87160691b3c96046c0c678fe57c5bef761456 +# TODO update this sha when we upgrade to Go 1.5+ ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \ && (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \ @@ -102,17 +113,17 @@ RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint # install seccomp # this can be changed to the ubuntu package libseccomp-dev if dockerinit is removed, # we need libseccomp.a (which the package does not provide) for dockerinit -ENV SECCOMP_VERSION v2.2.3 +ENV SECCOMP_VERSION 2.2.3 RUN set -x \ - && export SECCOMP_PATH=$(mktemp -d) \ - && git clone https://github.com/seccomp/libseccomp.git "$SECCOMP_PATH" \ + && export SECCOMP_PATH="$(mktemp -d)" \ + && curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \ + | tar -xzC "$SECCOMP_PATH" --strip-components=1 \ && ( \ cd "$SECCOMP_PATH" \ - && git checkout "$SECCOMP_VERSION" \ - && ./autogen.sh \ - && ./configure --prefix=/usr \ + && ./configure --prefix=/usr/local \ && make \ && make install \ + && ldconfig \ ) \ && rm -rf "$SECCOMP_PATH" @@ -127,29 +138,21 @@ RUN set -x \ && rm -rf "$GOPATH" # Install notary server -# commented Notary temporary as we are waiting for an update of jose2go: https://github.com/docker/notary/issues/239 -# -# ENV NOTARY_COMMIT 8e8122eb5528f621afcd4e2854c47302f17392f7 -# RUN set -x \ -# && export GOPATH="$(mktemp -d)" \ -# && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ -# && (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_COMMIT") \ -# && GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \ -# go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \ -# && rm -rf "$GOPATH" +ENV NOTARY_COMMIT f211b1826dde5fc8c117ccff9bb04ae458a8e3d0 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ + && (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_COMMIT") \ + && GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \ + go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \ + && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 139850f3f3b17357bab5ba3edfb745fb14043764 +ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ - && git checkout -q $DOCKER_PY_COMMIT - -# Setup s3cmd config -RUN { \ - echo '[default]'; \ - echo 'access_key=$AWS_ACCESS_KEY'; \ - echo 'secret_key=$AWS_SECRET_KEY'; \ - } > ~/.s3cfg + && git checkout -q $DOCKER_PY_COMMIT \ + && pip install -r test-requirements.txt # Set user.email so crosbymichael's in-container merge commits go smoothly RUN git config --global user.email 'docker-dummy@example.com' @@ -171,16 +174,16 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - hypriot/armhf-busybox:latest@sha256:b0fc94dac9793ce3c35607b15012b4c7deca300963a7cc38ab440189ec81e2e7 \ - hypriot/armhf-hello-world:latest@sha256:b618ec0cc3acf683e8d77ad6c5ec81546cddde2036eda9a78f628effdeca74cd \ - hypriot/armhf-unshare:latest@sha256:8fede091760d2fb8b2d14cedffdd681c4575b02b1abeeb18dd79b754c62327db + armhf/busybox:latest@sha256:d98a7343ac750ffe387e3d514f8521ba69846c216778919b01414b8617cfb3d4 \ + armhf/debian:jessie@sha256:094687129906d2a43cb4e5946ea379b5619c9ca8e4e27b3ba28b40f237a4150c \ + armhf/hello-world:latest@sha256:161dcecea0225975b2ad5f768058212c1e0d39e8211098666ffa1ac74cfb7791 # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is) # Download man page generator RUN set -x \ && export GOPATH="$(mktemp -d)" \ - && git clone -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ - && git clone -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ + && git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ + && git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ && go get -v -d github.com/cpuguy83/go-md2man \ && go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \ && rm -rf "$GOPATH" @@ -195,12 +198,12 @@ RUN set -x \ && rm -rf "$GOPATH" # Build/install the tool for embedding resources in Windows binaries -ENV RSRC_COMMIT e48dbf1b7fc464a9e85fcec450dddf80816b76e0 +ENV RSRC_VERSION v2 RUN set -x \ - && git clone https://github.com/akavel/rsrc.git /go/src/github.com/akavel/rsrc \ - && cd /go/src/github.com/akavel/rsrc \ - && git checkout -q $RSRC_COMMIT \ - && go install -v + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \ + && go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \ + && rm -rf "$GOPATH" # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Dockerfile.ppc64le b/Dockerfile.ppc64le index 7520b5dd4d..9c88d535b2 100644 --- a/Dockerfile.ppc64le +++ b/Dockerfile.ppc64le @@ -1,10 +1,27 @@ -# This file describes the standard way to build Docker, using docker +# This file describes the standard way to build Docker on ppc64le, using docker # # Usage: # # # Assemble the full dev environment. This is slow the first time. # docker build -t docker -f Dockerfile.ppc64le . # +# # Mount your source in an interactive container for quick testing: +# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash +# +# # Run the test suite: +# docker run --privileged docker hack/make.sh test +# +# # Publish a release: +# docker run --privileged \ +# -e AWS_S3_BUCKET=baz \ +# -e AWS_ACCESS_KEY=foo \ +# -e AWS_SECRET_KEY=bar \ +# -e GPG_PASSPHRASE=gloubiboulga \ +# docker hack/release.sh +# +# Note: AppArmor used to mess with privileged mode, but this is no longer +# the case. Therefore, you don't have to disable it anymore. +# FROM ppc64le/gcc:5.3 @@ -12,37 +29,83 @@ FROM ppc64le/gcc:5.3 RUN apt-get update && apt-get install -y \ apparmor \ aufs-tools \ + automake \ + bash-completion \ btrfs-tools \ build-essential \ + createrepo \ curl \ + dpkg-sig \ git \ iptables \ jq \ net-tools \ libapparmor-dev \ libcap-dev \ + libltdl-dev \ libsqlite3-dev \ + libsystemd-journal-dev \ + libtool \ mercurial \ - parallel \ + pkg-config \ python-dev \ python-mock \ python-pip \ python-websocket \ + xfsprogs \ + tar \ --no-install-recommends -RUN rm -rf /usr/local/lvm2 -RUN git clone --no-checkout git://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout -q v2_02_103 -RUN curl -o /usr/local/lvm2/autoconf/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -RUN curl -o /usr/local/lvm2/autoconf/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' +# Get lvm2 source for compiling statically +ENV LVM2_VERSION 2.02.103 +RUN mkdir -p /usr/local/lvm2 \ + && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ + | tar -xzC /usr/local/lvm2 --strip-components=1 +# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags + +# fix platform enablement in lvm2 to support ppc64le properly +RUN set -e \ + && for f in config.guess config.sub; do \ + curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \ + done +# "arch.c:78:2: error: #error the arch code needs to know about your machine type" # Compile and install lvm2 RUN cd /usr/local/lvm2 \ - && ./configure --enable-static_link \ + && ./configure \ + --build="$(gcc -print-multiarch)" \ + --enable-static_link \ && make device-mapper \ && make install_device-mapper +# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# TODO install Go, using gccgo as GOROOT_BOOTSTRAP (Go 1.5+ supports ppc64le properly) +# possibly a ppc64le/golang image? + +ENV PATH /go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor +# This has been commented out and kept as reference because we don't support compiling with older Go anymore. +# ENV GOFMT_VERSION 1.3.3 +# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt + +# TODO update this sha when we upgrade to Go 1.5+ +ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9 +# Grab Go's cover tool for dead-simple code coverage testing +# Grab Go's vet tool for examining go code to find suspicious constructs +# and help prevent errors that the compiler might not catch +RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \ + && (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \ + && go install -v golang.org/x/tools/cmd/cover \ + && go install -v golang.org/x/tools/cmd/vet +# Grab Go's lint tool +ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f +RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \ + && (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \ + && go install -v github.com/golang/lint/golint + + +# Install registry ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd RUN set -x \ && export GOPATH="$(mktemp -d)" \ @@ -63,12 +126,15 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 47ab89ec2bd3bddf1221b856ffbaff333edeabb4 +ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ && git checkout -q $DOCKER_PY_COMMIT \ && pip install -r test-requirements.txt +# Set user.email so crosbymichael's in-container merge commits go smoothly +RUN git config --global user.email 'docker-dummy@example.com' + # Add an unprivileged user to be used for tests which need it RUN groupadd -r docker RUN useradd --create-home --gid docker unprivilegeduser @@ -77,12 +143,45 @@ VOLUME /var/lib/docker WORKDIR /go/src/github.com/docker/docker ENV DOCKER_BUILDTAGS apparmor selinux -ENV IMAGEREPO ppc64le +# Let us use a .bashrc file +RUN ln -sfv $PWD/.bashrc ~/.bashrc + +# Register Docker's bash completion. +RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker + +# Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - $IMAGEREPO/busybox:latest \ - $IMAGEREPO/hello-world:frozen \ - $IMAGEREPO/unshare:latest + ppc64le/busybox:latest@sha256:38bb82085248d5a3c24bd7a5dc146f2f2c191e189da0441f1c2ca560e3fc6f1b \ + ppc64le/debian:jessie@sha256:74e06e6506b23cf8abd00250782838b2d19910824d8e7eab3d14dc1845ea10c6 \ + ppc64le/hello-world:latest@sha256:186a40a9a02ca26df0b6c8acdfb8ac2f3ae6678996a838f977e57fac9d963974 +# see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is) + +# Download man page generator +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ + && git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ + && go get -v -d github.com/cpuguy83/go-md2man \ + && go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \ + && rm -rf "$GOPATH" + +# Download toml validator +ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \ + && (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \ + && go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \ + && rm -rf "$GOPATH" + +# Build/install the tool for embedding resources in Windows binaries +ENV RSRC_VERSION v2 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \ + && go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \ + && rm -rf "$GOPATH" # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Dockerfile.s390x b/Dockerfile.s390x index c035979d48..cf03021283 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -1,47 +1,111 @@ -# This file describes the standard way to build Docker, using docker +# This file describes the standard way to build Docker on s390x, using docker # # Usage: # # # Assemble the full dev environment. This is slow the first time. # docker build -t docker -f Dockerfile.s390x . # +# # Mount your source in an interactive container for quick testing: +# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash +# +# # Run the test suite: +# docker run --privileged docker hack/make.sh test +# +# # Publish a release: +# docker run --privileged \ +# -e AWS_S3_BUCKET=baz \ +# -e AWS_ACCESS_KEY=foo \ +# -e AWS_SECRET_KEY=bar \ +# -e GPG_PASSPHRASE=gloubiboulga \ +# docker hack/release.sh +# +# Note: AppArmor used to mess with privileged mode, but this is no longer +# the case. Therefore, you don't have to disable it anymore. +# -FROM s390x/gcc:5.2 +FROM s390x/gcc:5.3 # Packaged dependencies RUN apt-get update && apt-get install -y \ apparmor \ aufs-tools \ + automake \ + bash-completion \ btrfs-tools \ build-essential \ + createrepo \ curl \ + dpkg-sig \ git \ iptables \ jq \ net-tools \ libapparmor-dev \ libcap-dev \ + libltdl-dev \ libsqlite3-dev \ + libsystemd-journal-dev \ + libtool \ mercurial \ - parallel \ + pkg-config \ python-dev \ python-mock \ python-pip \ python-websocket \ + xfsprogs \ + tar \ --no-install-recommends # Get lvm2 source for compiling statically -RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 +ENV LVM2_VERSION 2.02.103 +RUN mkdir -p /usr/local/lvm2 \ + && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ + | tar -xzC /usr/local/lvm2 --strip-components=1 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags +# fix platform enablement in lvm2 to support s390x properly +RUN set -e \ + && for f in config.guess config.sub; do \ + curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \ + done +# "arch.c:78:2: error: #error the arch code needs to know about your machine type" + # Compile and install lvm2 RUN cd /usr/local/lvm2 \ - && ./configure --enable-static_link \ + && ./configure \ + --build="$(gcc -print-multiarch)" \ + --enable-static_link \ && make device-mapper \ && make install_device-mapper +# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# Note: Go comes from the base image (gccgo, specifically) +# We can't compile Go proper because s390x isn't an officially supported architecture yet. + +ENV PATH /go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor +# This has been commented out and kept as reference because we don't support compiling with older Go anymore. +# ENV GOFMT_VERSION 1.3.3 +# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt + +# TODO update this sha when we upgrade to Go 1.5+ +ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9 +# Grab Go's cover tool for dead-simple code coverage testing +# Grab Go's vet tool for examining go code to find suspicious constructs +# and help prevent errors that the compiler might not catch +RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \ + && (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \ + && go install -v golang.org/x/tools/cmd/cover \ + && go install -v golang.org/x/tools/cmd/vet +# Grab Go's lint tool +ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f +RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \ + && (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \ + && go install -v github.com/golang/lint/golint + + +# Install registry ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd RUN set -x \ && export GOPATH="$(mktemp -d)" \ @@ -62,12 +126,15 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 47ab89ec2bd3bddf1221b856ffbaff333edeabb4 +ENV DOCKER_PY_COMMIT 57512760c83fbe41302891aa51e34a86f4db74de RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ && git checkout -q $DOCKER_PY_COMMIT \ && pip install -r test-requirements.txt +# Set user.email so crosbymichael's in-container merge commits go smoothly +RUN git config --global user.email 'docker-dummy@example.com' + # Add an unprivileged user to be used for tests which need it RUN groupadd -r docker RUN useradd --create-home --gid docker unprivilegeduser @@ -76,12 +143,45 @@ VOLUME /var/lib/docker WORKDIR /go/src/github.com/docker/docker ENV DOCKER_BUILDTAGS apparmor selinux -ENV IMAGEREPO s390x +# Let us use a .bashrc file +RUN ln -sfv $PWD/.bashrc ~/.bashrc + +# Register Docker's bash completion. +RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker + +# Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - $IMAGEREPO/busybox:latest \ - $IMAGEREPO/hello-world:frozen \ - $IMAGEREPO/unshare:latest + s390x/busybox:latest@sha256:dd61522c983884a66ed72d60301925889028c6d2d5e0220a8fe1d9b4c6a4f01b \ + s390x/debian:jessie@sha256:3c478e199f60c877c00306356267798d32727dc3cd38512cdb4b060659ea9d20 \ + s390x/hello-world:latest@sha256:780d80b3a7677c3788c0d5cd9168281320c8d4a6d9183892d8ee5cdd610f5699 +# see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is) + +# Download man page generator +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \ + && git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \ + && go get -v -d github.com/cpuguy83/go-md2man \ + && go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \ + && rm -rf "$GOPATH" + +# Download toml validator +ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \ + && (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \ + && go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \ + && rm -rf "$GOPATH" + +# Build/install the tool for embedding resources in Windows binaries +ENV RSRC_VERSION v2 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone --depth 1 -b "$RSRC_VERSION" https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \ + && go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \ + && rm -rf "$GOPATH" # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Makefile b/Makefile index 17ab01b965..d0c6092420 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,27 @@ .PHONY: all binary build cross default docs docs-build docs-shell shell test test-docker-py test-integration-cli test-unit validate # get OS/Arch of docker engine -DOCKER_ENGINE_OSARCH = $(shell docker version | grep 'OS/Arch' | tail -1 | cut -d':' -f2 | tr -d '[[:space:]]') +DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH:+$$DOCKER_CLIENT_OSARCH}') # default for linux/amd64 and others -DOCKER_FILE = Dockerfile +DOCKERFILE := Dockerfile # switch to different Dockerfile for linux/arm -ifeq ($(DOCKER_ENGINE_OSARCH),linux/arm) - DOCKER_FILE = Dockerfile.armhf +ifeq ($(DOCKER_OSARCH), linux/arm) + DOCKERFILE := Dockerfile.armhf +else +ifeq ($(DOCKER_OSARCH), linux/arm64) + # TODO .arm64 + DOCKERFILE := Dockerfile.armhf +else +ifeq ($(DOCKER_OSARCH), linux/ppc64le) + DOCKERFILE := Dockerfile.ppc64le +else +ifeq ($(DOCKER_OSARCH), linux/s390x) + DOCKERFILE := Dockerfile.s390x endif -export DOCKER_FILE +endif +endif +endif +export DOCKERFILE # env vars passed through directly to Docker's build scripts # to allow things like `make DOCKER_CLIENTONLY=1 binary` easily @@ -18,7 +31,7 @@ DOCKER_ENVS := \ -e DOCKER_CLIENTONLY \ -e DOCKER_DEBUG \ -e DOCKER_EXPERIMENTAL \ - -e DOCKER_FILE \ + -e DOCKERFILE \ -e DOCKER_GRAPHDRIVER \ -e DOCKER_REMAP_ROOT \ -e DOCKER_STORAGE_OPTS \ @@ -60,7 +73,7 @@ binary: build $(DOCKER_RUN_DOCKER) hack/make.sh binary build: bundles - docker build -t "$(DOCKER_IMAGE)" -f $(DOCKER_FILE) . + docker build -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles diff --git a/hack/make/.detect-daemon-osarch b/hack/make/.detect-daemon-osarch new file mode 100644 index 0000000000..582f69b385 --- /dev/null +++ b/hack/make/.detect-daemon-osarch @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Retrieve OS/ARCH of docker daemon, eg. linux/amd64 +export DOCKER_ENGINE_OSARCH="$(docker version |tac|tac| awk ' + $1 == "Client:" { server = 0; next } + $1 == "Server:" { server = 1; next } + server && $1 == "OS/Arch:" { print $2; exit } +')" +export DOCKER_ENGINE_GOOS="${DOCKER_ENGINE_OSARCH%/*}" +export DOCKER_ENGINE_GOARCH="${DOCKER_ENGINE_OSARCH##*/}" + +# and the client, just in case +export DOCKER_CLIENT_OSARCH="$(docker version |tac|tac| awk ' + $1 == "Client:" { client = 1; next } + $1 == "Server:" { client = 0; next } + client && $1 == "OS/Arch:" { print $2; exit } +')" diff --git a/hack/make/.ensure-frozen-images b/hack/make/.ensure-frozen-images index 6c4ae7f22c..6e0036c986 100644 --- a/hack/make/.ensure-frozen-images +++ b/hack/make/.ensure-frozen-images @@ -1,38 +1,32 @@ #!/bin/bash set -e -# image lists for different archs that should match what's in the Dockerfile (minus the explicit images IDs) +# image list should match what's in the Dockerfile (minus the explicit images IDs) +images=( + busybox:latest + debian:jessie + hello-world:latest +) + +imagePrefix= case "$DOCKER_ENGINE_OSARCH" in linux/arm) - images=( - hypriot/armhf-busybox:latest - hypriot/armhf-hello-world:latest - hypriot/armhf-unshare:latest - ) + imagePrefix='armhf' ;; linux/ppc64le) - images=( - ppc64le/busybox:latest - ppc64le/hello-world:frozen - ppc64le/unshare:latest - ) + imagePrefix='ppc64le' ;; linux/s390x) - images=( - s390x/busybox:latest - s390x/hello-world:frozen - s390x/unshare:latest - ) - ;; - *) - images=( - busybox:latest - debian:jessie - hello-world:latest - ) + imagePrefix='s390x' ;; esac +if [ "$imagePrefix" ]; then + for (( i = 0; i < ${#images[@]}; i++ )); do + images[$i]="$imagePrefix/${images[$i]}" + done +fi + if ! docker inspect "${images[@]}" &> /dev/null; then hardCodedDir='/docker-frozen-images' if [ -d "$hardCodedDir" ]; then @@ -60,7 +54,7 @@ if ! docker inspect "${images[@]}" &> /dev/null; then inCont = 0; } } - ' ${DOCKER_FILE:="Dockerfile"} | sh -x + ' "${DOCKERFILE:=Dockerfile}" | sh -x # Do not use a subshell for the following command. Windows CI # runs bash 3.x so will not trap an error in a subshell. # http://stackoverflow.com/questions/22630363/how-does-set-e-work-with-subshells @@ -68,35 +62,18 @@ if ! docker inspect "${images[@]}" &> /dev/null; then fi fi -# tag images to ensure that all integrations work with the defined image names -# then remove original tags as these make problems with later tests (e.g., TestInspectApiImageResponse) -case "$DOCKER_ENGINE_OSARCH" in - linux/arm) - docker tag hypriot/armhf-busybox:latest busybox:latest - docker tag hypriot/armhf-hello-world:latest hello-world:frozen - docker tag hypriot/armhf-unshare:latest jess/unshare:latest - docker rmi hypriot/armhf-busybox:latest - docker rmi hypriot/armhf-hello-world:latest - docker rmi hypriot/armhf-unshare:latest - ;; - linux/ppc64le) - docker tag ppc64le/busybox:latest busybox:latest - docker tag ppc64le/hello-world:frozen hello-world:frozen - docker tag ppc64le/unshare:latest jess/unshare:latest - docker rmi ppc64le/busybox:latest - docker rmi ppc64le/hello-world:frozen - docker rmi ppc64le/unshare:latest - ;; - linux/s390x) - docker tag s390x/busybox:latest busybox:latest - docker tag s390x/hello-world:frozen hello-world:frozen - docker tag s390x/unshare:latest jess/unshare:latest - docker rmi s390x/busybox:latest - docker rmi s390x/hello-world:frozen - docker rmi s390x/unshare:latest - ;; - *) - docker tag hello-world:latest hello-world:frozen - docker rmi hello-world:latest - ;; -esac +if [ "$imagePrefix" ]; then + for image in "${images[@]}"; do + target="${image#$imagePrefix/}" + if [ "$target" != "$image" ]; then + # tag images to ensure that all integrations work with the defined image names + docker tag "$image" "$target" + # then remove original tags as these make problems with later tests (e.g., TestInspectApiImageResponse) + docker rmi "$image" + fi + done +fi + +# explicitly rename "hello-world:latest" to ":frozen" for the test that uses it +docker tag hello-world:latest hello-world:frozen +docker rmi hello-world:latest diff --git a/hack/make/.integration-daemon-setup b/hack/make/.integration-daemon-setup index 4703685ca4..97324862db 100644 --- a/hack/make/.integration-daemon-setup +++ b/hack/make/.integration-daemon-setup @@ -1,12 +1,7 @@ #!/bin/bash +set -e -# Retrieve OS/ARCH of docker daemon, eg. linux/amd64 -export DOCKER_ENGINE_OSARCH=$(docker version | grep 'OS/Arch' | tail -1 | cut -d':' -f2 | tr -d '[[:space:]]') -# Retrieve OS of docker daemon, eg. linux -export DOCKER_ENGINE_GOOS=$(echo $DOCKER_ENGINE_OSARCH | cut -d'/' -f1) -# Retrieve ARCH of docker daemon, eg. amd64 -export DOCKER_ENGINE_GOARCH=$(echo $DOCKER_ENGINE_OSARCH | cut -d'/' -f2) - +bundle .detect-daemon-osarch bundle .ensure-emptyfs bundle .ensure-frozen-images bundle .ensure-httpserver