Merge pull request #8413 from thaJeztah/optimize_dockerfile

Dockerfile: optimize for caching
This commit is contained in:
Maria Bermudez 2019-03-06 20:19:56 -08:00 committed by GitHub
commit 5d111f85dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 100 additions and 108 deletions

View File

@ -2,81 +2,92 @@
# from the master branch of https://github.com/docker/docker.github.io
#
# Here is the sequence:
# 1. Set up the build
# 2. Fetch upstream resources
# 3. Build static HTML from master
# 4. Reset to clean tiny nginx image
# 5. Copy Nginx config and archive HTML, which don't change often and can be cached
# 6. Copy static HTML from previous build stage (step 3)
# 1. Set up base stages for building and deploying
# 2. Collect and build the archived documentation
# 3. Collect and build the reference documentation (from upstream resources)
# 4. Build static HTML from the current branch
# 5. Build the final image, combining the archives, reference docs, and
# current version of the documentation
#
# When the image is run, it starts Nginx and serves the docs at port 4000
# Get basic configs and Jekyll env
FROM docs/docker.github.io:docs-builder AS builder
# Set the target again
ENV TARGET=/usr/share/nginx/html
# Set the source directory to md_source
ENV SOURCE=md_source
# Get the current docs from the checked out branch
# ${SOURCE} will contain a directory for each archive
COPY . ${SOURCE}
####### START UPSTREAM RESOURCES ########
# Set vars used by fetch-upstream-resources.sh script
## Branch to pull from, per ref doc
## To get master from svn the svn branch needs to be 'trunk'. To get a branch from svn it needs to be 'branches/branchname'
# Engine
ENV ENGINE_SVN_BRANCH="branches/18.09.x"
ENV ENGINE_BRANCH="18.09.x"
ARG ENGINE_BRANCH="18.09.x"
# Distribution
ENV DISTRIBUTION_SVN_BRANCH="branches/release/2.6"
ENV DISTRIBUTION_BRANCH="release/2.6"
# Fetch upstream resources
RUN bash ./${SOURCE}/_scripts/fetch-upstream-resources.sh ${SOURCE}
####### END UPSTREAM RESOURCES ########
ARG DISTRIBUTION_BRANCH="release/2.6"
# Build the static HTML, now that everything is in place
###
# Set up base stages for building and deploying
###
RUN jekyll build -s ${SOURCE} -d ${TARGET} --config ${SOURCE}/_config.yml
# Get basic configs and Jekyll env
FROM docs/docker.github.io:docs-builder AS builderbase
ENV TARGET=/usr/share/nginx/html
WORKDIR /usr/src/app/md_source/
# Fix up some links, don't touch the archives
RUN find ${TARGET} -type f -name '*.html' | grep -vE "v[0-9]+\." | while read i; do sed -i 's#href="https://docs.docker.com/#href="/#g' "$i"; done
# Set vars used by fetch-upstream-resources.sh script
# Branch to pull from, per ref doc. To get master from svn the svn branch needs
# to be 'trunk'. To get a branch from svn it needs to be 'branches/branchname'
ARG ENGINE_BRANCH
ENV ENGINE_BRANCH=${ENGINE_BRANCH}
ENV ENGINE_SVN_BRANCH=branches/${ENGINE_BRANCH}
ARG DISTRIBUTION_BRANCH
ENV DISTRIBUTION_BRANCH=${DISTRIBUTION_BRANCH}
ENV DISTRIBUTION_SVN_BRANCH=branches/${DISTRIBUTION_BRANCH}
# BUILD OF MASTER DOCS IS NOW DONE!
# Reset to alpine so we don't get any docs source or extra apps
FROM nginx:alpine
# Set the target again
FROM nginx:alpine AS deploybase
ENV TARGET=/usr/share/nginx/html
# Get the nginx config from the nginx-onbuild image
# This hardly ever changes so should usually be cached
COPY --from=docs/docker.github.io:nginx-onbuild /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf
# Get all the archive static HTML and put it into place
# Go oldest-to-newest to take advantage of the fact that we change older
# archives less often than new ones.
# To add a new archive, add it here
# AND ALSO edit _data/docsarchives/archives.yaml to add it to the drop-down
# Set the default command to serve the static HTML site
CMD echo -e "Docker docs are viewable at:\nhttp://0.0.0.0:4000"; exec nginx -g 'daemon off;'
# Build the archived docs
# these docs barely change, so can be cached
FROM deploybase AS archives
# Get all the archive static HTML and put it into place. To add a new archive,
# add it here, and ALSO edit _data/docsarchives/archives.yaml to add it to the drop-down
COPY --from=docs/docker.github.io:v17.03 ${TARGET} ${TARGET}
COPY --from=docs/docker.github.io:v17.06 ${TARGET} ${TARGET}
COPY --from=docs/docker.github.io:v17.09 ${TARGET} ${TARGET}
COPY --from=docs/docker.github.io:v17.12 ${TARGET} ${TARGET}
COPY --from=docs/docker.github.io:v18.03 ${TARGET} ${TARGET}
# Get the built docs output from the previous build stage
# This ordering means all previous layers can come from cache unless an archive
# changes
# Fetch upstream resources (reference documentation)
# Only add the files that are needed to build these reference docs, so that
# these docs are only rebuilt if changes were made to the configuration.
FROM builderbase AS upstream-resources
COPY ./_scripts/fetch-upstream-resources.sh ./_scripts/
COPY ./_config.yml .
COPY ./_data/toc.yaml ./_data/
RUN bash ./_scripts/fetch-upstream-resources.sh .
COPY --from=builder ${TARGET} ${TARGET}
# Serve the site (target), which is now all static HTML
CMD echo -e "Docker docs are viewable at:\nhttp://0.0.0.0:4000"; exec nginx -g 'daemon off;'
# Build the current docs from the checked out branch
FROM builderbase AS current
COPY . .
COPY --from=upstream-resources /usr/src/app/md_source/. ./
# Build the static HTML, now that everything is in place
RUN jekyll build -d ${TARGET}
# Fix up some links, don't touch the archives
RUN find ${TARGET} -type f -name '*.html' | grep -vE "v[0-9]+\." | while read i; do sed -i 's#href="https://docs.docker.com/#href="/#g' "$i"; done
# Docs with archives (for deploy)
FROM archives AS deploy
# Add the current version of the docs
COPY --from=current ${TARGET} ${TARGET}

View File

@ -29,28 +29,11 @@ while getopts ":hl" opt; do
done
# Do some sanity-checking to make sure we are running this from the right place
if [ $LOCAL -eq 1 ]; then
SOURCE="."
if ! [ -f _config.yml ]; then
echo "Could not find _config.yml. We may not be in the right place. Bailing."
exit 1
fi
else
SOURCE="md_source"
if ! [ -d md_source ]; then
echo "Could not find md_source directory. We may not be running in the right place. Bailing."
exit 1
fi
if ! [ -f _config.yml ]; then
echo "Could not find _config.yml. We may not be in the right place. Bailing."
exit 1
fi
# Reasonable default to find the Markdown files
if [ -z "$SOURCE" ]; then
echo "No source passed in, assuming md_source/..."
SOURCE="md_source"
fi
echo "Operating on contents of $SOURCE"
# Parse some variables from _config.yml and make them available to this script
# This only finds top-level variables with _version in them that don't have any
# leading space. This is brittle!
@ -61,10 +44,10 @@ while read i; do
varvalue=$(echo "$i" | sed 's/"//g' | awk -F ':' {'print $2'} | tr -d '[:space:]')
echo "Setting \$${varname} to $varvalue"
declare "$varname=$varvalue"
done < <(cat ${SOURCE}/_config.yml |grep '_version:' |grep '^[a-z].*')
done < <(cat ./_config.yml |grep '_version:' |grep '^[a-z].*')
# Replace variable in toc.yml with value from above
sedi "s/{{ site.latest_engine_api_version }}/$latest_engine_api_version/g" ${SOURCE}/_data/toc.yaml
sedi "s/{{ site.latest_engine_api_version }}/$latest_engine_api_version/g" ./_data/toc.yaml
# Engine stable
ENGINE_SVN_BRANCH="branches/18.09"
@ -75,17 +58,17 @@ DISTRIBUTION_SVN_BRANCH="branches/release/2.6"
DISTRIBUTION_BRANCH="release/2.6"
# Directories to get via SVN. We use this because you can't use git to clone just a portion of a repository
svn co https://github.com/docker/docker-ce/"$ENGINE_SVN_BRANCH"/components/cli/docs/extend ${SOURCE}/engine/extend || (echo "Failed engine/extend download" && exit -1)
svn co https://github.com/docker/docker-ce/"$ENGINE_SVN_BRANCH"/components/engine/docs/api ${SOURCE}/engine/api || (echo "Failed engine/api download" && exit -1) # This will only get you the old API MD files 1.18 through 1.24
svn co https://github.com/docker/distribution/"$DISTRIBUTION_SVN_BRANCH"/docs/spec ${SOURCE}/registry/spec || (echo "Failed registry/spec download" && exit -1)
svn co https://github.com/docker/compliance/trunk/docs/compliance ${SOURCE}/compliance || (echo "Failed docker/compliance download" && exit -1)
svn co https://github.com/docker/docker-ce/"$ENGINE_SVN_BRANCH"/components/cli/docs/extend ./engine/extend || (echo "Failed engine/extend download" && exit 1)
svn co https://github.com/docker/docker-ce/"$ENGINE_SVN_BRANCH"/components/engine/docs/api ./engine/api || (echo "Failed engine/api download" && exit 1) # This will only get you the old API MD files 1.18 through 1.24
svn co https://github.com/docker/distribution/"$DISTRIBUTION_SVN_BRANCH"/docs/spec ./registry/spec || (echo "Failed registry/spec download" && exit 1)
svn co https://github.com/docker/compliance/trunk/docs/compliance ./compliance || (echo "Failed docker/compliance download" && exit 1)
# Get the Library docs
svn co https://github.com/docker-library/docs/trunk ${SOURCE}/_samples/library || (echo "Failed library download" && exit -1)
svn co https://github.com/docker-library/docs/trunk ./_samples/library || (echo "Failed library download" && exit 1)
# Remove symlinks to maintainer.md because they break jekyll and we don't use em
find ${SOURCE}/_samples/library -maxdepth 9 -type l -delete
find ./_samples/library -maxdepth 9 -type l -delete
# Loop through the README.md files, turn them into rich index.md files
FILES=$(find ${SOURCE}/_samples/library -type f -name 'README.md')
FILES=$(find ./_samples/library -type f -name 'README.md')
for f in $FILES
do
curdir=$(dirname "${f}")
@ -118,7 +101,7 @@ do
echo GitHub repo: \["${gitrepo}"\]\("${gitrepo}"\)\{: target="_blank"\} >> ${curdir}/front-matter.txt
echo >> ${curdir}/front-matter.txt
fi
cat ${curdir}/front-matter.txt ${SOURCE}/_samples/boilerplate.txt > ${curdir}/header.txt
cat ${curdir}/front-matter.txt ./_samples/boilerplate.txt > ${curdir}/header.txt
echo {% raw %} >> ${curdir}/header.txt
cat ${curdir}/header.txt ${curdir}/README.md > ${curdir}/index.md
echo {% endraw %} >> ${curdir}/index.md
@ -128,41 +111,39 @@ done
# Get the Engine APIs that are in Swagger
# Be careful with the locations on Github for these
wget -O ${SOURCE}/engine/api/v1.25/swagger.yaml https://raw.githubusercontent.com/docker/docker/v1.13.0/api/swagger.yaml || (echo "Failed 1.25 swagger download" && exit -1)
wget -O ${SOURCE}/engine/api/v1.26/swagger.yaml https://raw.githubusercontent.com/docker/docker/v17.03.0-ce/api/swagger.yaml || (echo "Failed 1.26 swagger download" && exit -1)
wget -O ${SOURCE}/engine/api/v1.27/swagger.yaml https://raw.githubusercontent.com/docker/docker/v17.03.1-ce/api/swagger.yaml || (echo "Failed 1.27 swagger download" && exit -1)
# Get the Edge API Swagger
# When you change this you need to make sure to copy the previous
# directory into a new one in the docs git and change the index.html
wget -O ${SOURCE}/engine/api/v1.28/swagger.yaml https://raw.githubusercontent.com/docker/docker/v17.04.0-ce/api/swagger.yaml || (echo "Failed 1.28 swagger download or the 1.28 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.29/swagger.yaml https://raw.githubusercontent.com/docker/docker/17.05.x/api/swagger.yaml || (echo "Failed 1.29 swagger download or the 1.29 directory doesn't exist" && exit -1)
wget --directory-prefix=./engine/api/v1.25/ https://raw.githubusercontent.com/docker/docker/v1.13.0/api/swagger.yaml || (echo "Failed 1.25 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.26/ https://raw.githubusercontent.com/docker/docker/v17.03.0-ce/api/swagger.yaml || (echo "Failed 1.26 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.27/ https://raw.githubusercontent.com/docker/docker/v17.03.1-ce/api/swagger.yaml || (echo "Failed 1.27 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.28/ https://raw.githubusercontent.com/docker/docker/v17.04.0-ce/api/swagger.yaml || (echo "Failed 1.28 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.29/ https://raw.githubusercontent.com/docker/docker/17.05.x/api/swagger.yaml || (echo "Failed 1.29 swagger download" && exit 1)
# New location for swagger.yaml for 17.06+
wget -O ${SOURCE}/engine/api/v1.30/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.06/components/engine/api/swagger.yaml || (echo "Failed 1.30 swagger download or the 1.30 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.31/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.07/components/engine/api/swagger.yaml || (echo "Failed 1.31 swagger download or the 1.31 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.32/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.09/components/engine/api/swagger.yaml || (echo "Failed 1.32 swagger download or the 1.32 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.33/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.10/components/engine/api/swagger.yaml || (echo "Failed 1.33 swagger download or the 1.33 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.34/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.11/components/engine/api/swagger.yaml || (echo "Failed 1.34 swagger download or the 1.34 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.35/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/17.12/components/engine/api/swagger.yaml || (echo "Failed 1.35 swagger download or the 1.35 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.36/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/18.02/components/engine/api/swagger.yaml || (echo "Failed 1.36 swagger download or the 1.36 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.37/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/18.03/components/engine/api/swagger.yaml || (echo "Failed 1.37 swagger download or the 1.37 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.38/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/18.06/components/engine/api/swagger.yaml || (echo "Failed 1.38 swagger download or the 1.38 directory doesn't exist" && exit -1)
wget -O ${SOURCE}/engine/api/v1.39/swagger.yaml https://raw.githubusercontent.com/docker/docker-ce/18.09/components/engine/api/swagger.yaml || (echo "Failed 1.39 swagger download or the 1.39 directory doesn't exist" && exit -1)
wget --directory-prefix=./engine/api/v1.30/ https://raw.githubusercontent.com/docker/docker-ce/17.06/components/engine/api/swagger.yaml || (echo "Failed 1.30 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.31/ https://raw.githubusercontent.com/docker/docker-ce/17.07/components/engine/api/swagger.yaml || (echo "Failed 1.31 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.32/ https://raw.githubusercontent.com/docker/docker-ce/17.09/components/engine/api/swagger.yaml || (echo "Failed 1.32 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.33/ https://raw.githubusercontent.com/docker/docker-ce/17.10/components/engine/api/swagger.yaml || (echo "Failed 1.33 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.34/ https://raw.githubusercontent.com/docker/docker-ce/17.11/components/engine/api/swagger.yaml || (echo "Failed 1.34 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.35/ https://raw.githubusercontent.com/docker/docker-ce/17.12/components/engine/api/swagger.yaml || (echo "Failed 1.35 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.36/ https://raw.githubusercontent.com/docker/docker-ce/18.02/components/engine/api/swagger.yaml || (echo "Failed 1.36 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.37/ https://raw.githubusercontent.com/docker/docker-ce/18.03/components/engine/api/swagger.yaml || (echo "Failed 1.37 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.38/ https://raw.githubusercontent.com/docker/docker-ce/18.06/components/engine/api/swagger.yaml || (echo "Failed 1.38 swagger download" && exit 1)
wget --directory-prefix=./engine/api/v1.39/ https://raw.githubusercontent.com/docker/docker-ce/18.09/components/engine/api/swagger.yaml || (echo "Failed 1.39 swagger download" && exit 1)
# Get dockerd.md from upstream
wget -O ${SOURCE}/engine/reference/commandline/dockerd.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/commandline/dockerd.md || (echo "Failed to fetch stable dockerd.md" && exit -1)
wget --directory-prefix=./engine/reference/commandline/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/commandline/dockerd.md || (echo "Failed to fetch stable dockerd.md" && exit 1)
# Get a few one-off files that we use directly from upstream
wget -O ${SOURCE}/engine/reference/builder.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/builder.md || (echo "Failed engine/reference/builder.md download" && exit -1)
wget -O ${SOURCE}/engine/reference/run.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/run.md || (echo "Failed engine/reference/run.md download" && exit -1)
wget --directory-prefix=./engine/reference/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/builder.md || (echo "Failed engine/reference/builder.md download" && exit 1)
wget --directory-prefix=./engine/reference/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/run.md || (echo "Failed engine/reference/run.md download" && exit 1)
# Adjust this one when Edge != Stable
wget -O ${SOURCE}/edge/engine/reference/run.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/run.md || (echo "Failed engine/reference/run.md download" && exit -1)
wget -O ${SOURCE}/engine/reference/commandline/cli.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/commandline/cli.md || (echo "Failed engine/reference/commandline/cli.md download" && exit -1)
wget -O ${SOURCE}/engine/deprecated.md https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/deprecated.md || (echo "Failed engine/deprecated.md download" && exit -1)
wget -O ${SOURCE}/registry/configuration.md https://raw.githubusercontent.com/docker/distribution/"$DISTRIBUTION_BRANCH"/docs/configuration.md || (echo "Failed registry/configuration.md download" && exit -1)
wget --directory-prefix=./edge/engine/reference/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/run.md || (echo "Failed engine/reference/run.md download" && exit 1)
wget --directory-prefix=./engine/reference/commandline/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/reference/commandline/cli.md || (echo "Failed engine/reference/commandline/cli.md download" && exit 1)
wget --directory-prefix=./engine/ https://raw.githubusercontent.com/docker/docker-ce/"$ENGINE_BRANCH"/components/cli/docs/deprecated.md || (echo "Failed engine/deprecated.md download" && exit 1)
wget --directory-prefix=./registry/ https://raw.githubusercontent.com/docker/distribution/"$DISTRIBUTION_BRANCH"/docs/configuration.md || (echo "Failed registry/configuration.md download" && exit 1)
# Remove things we don't want in the build
rm ${SOURCE}/registry/spec/api.md.tmpl
rm -rf ${SOURCE}/apidocs/cloud-api-source
rm -rf ${SOURCE}/tests
rm ${SOURCE}/_samples/library/index.md
rm ./registry/spec/api.md.tmpl
rm -rf ./apidocs/cloud-api-source
rm -rf ./tests
rm ./_samples/library/index.md