diff --git a/Dockerfile b/Dockerfile index 5f779e227f..bf4a0cf956 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,15 +12,10 @@ ARG BUNDLER_VERSION=2.3.13 ARG JEKYLL_ENV=development ARG DOMAIN=docs.docker.com -ARG ENGINE_BRANCH="20.10" -ARG DISTRIBUTION_BRANCH="release/2.7" -ARG COMPOSE_CLI_BRANCH="main" -ARG EXTENSIONS_SDK_BRANCH="main" - # Base stage for building FROM ruby:${RUBY_VERSION}-alpine AS base WORKDIR /src -RUN apk add --no-cache bash build-base git subversion wget +RUN apk add --no-cache bash build-base git # Gem stage will install bundler used as dependency manager # for our dependencies in Gemfile for Jekyll @@ -45,27 +40,14 @@ RUN bundle update \ FROM scratch AS vendor COPY --from=vendored /out / -# Fetch upstream resources (reference documentation) -# Only add the files that are needed to build these reference docs, so that these -# docs are only rebuilt if changes were made to ENGINE_BRANCH or DISTRIBUTION_BRANCH. -FROM base AS upstream-resources -WORKDIR /out -COPY ./_scripts/fetch-upstream-resources.sh ./_scripts/ -ARG ENGINE_BRANCH -ARG DISTRIBUTION_BRANCH -ARG COMPOSE_CLI_BRANCH -ARG EXTENSIONS_SDK_BRANCH -RUN ./_scripts/fetch-upstream-resources.sh . - # Build the static HTML for the current docs. # After building with jekyll, fix up some links FROM gem AS generate ARG JEKYLL_ENV ARG DOMAIN ENV TARGET=/out -COPY . . -COPY --from=upstream-resources /out . -RUN --mount=type=cache,target=/src/.jekyll-cache <= 0.1.0' if Gem.win_platform? + gem 'jekyll' gem 'jekyll-redirect-from' gem 'jekyll-relative-links' gem 'jekyll-sitemap' + +gem 'archive-zip' gem 'html-proofer' +gem 'octopress-hooks' +gem 'rake' diff --git a/Gemfile.lock b/Gemfile.lock index af016bebda..b9b493f4d5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,6 +3,8 @@ GEM specs: addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) + archive-zip (0.12.0) + io-like (~> 0.3.0) colorator (1.1.0) concurrent-ruby (1.1.10) em-websocket (0.5.3) @@ -24,6 +26,7 @@ GEM http_parser.rb (0.8.0) i18n (1.10.0) concurrent-ruby (~> 1.0) + io-like (0.3.1) jekyll (4.2.2) addressable (~> 2.4) colorator (~> 1.0) @@ -58,14 +61,17 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) - nokogiri (1.13.5-x86_64-linux) + nokogiri (1.13.6-x86_64-linux) racc (~> 1.4) + octopress-hooks (2.6.2) + jekyll (>= 2.0) parallel (1.22.1) pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (4.0.7) racc (1.6.0) rainbow (3.1.1) + rake (13.0.6) rb-fsevent (0.11.1) rb-inotify (0.10.1) ffi (~> 1.0) @@ -87,11 +93,14 @@ PLATFORMS x86_64-linux DEPENDENCIES + archive-zip html-proofer jekyll jekyll-redirect-from jekyll-relative-links jekyll-sitemap + octopress-hooks + rake tzinfo-data BUNDLED WITH diff --git a/_config.yml b/_config.yml index ebdbd92310..64d8aca0a0 100644 --- a/_config.yml +++ b/_config.yml @@ -104,3 +104,62 @@ defaults: path: engine/reference/commandline values: skip_read_time: true + +# Fetch upstream resources (reference documentation) used by _plugins/fetch_remote.rb +# - repo is the GitHub repository to fetch from +# - ref the Git reference +# - paths is a list to the resources within the remote repository +# - dest is the destination path within the working tree +# - src is a list of glob source paths within the remote repository +fetch-remote: + - repo: "https://github.com/docker/cli" + ref: "20.10" + paths: + - dest: "engine/extend" + src: + - "docs/extend/**" + - dest: "engine" + src: + - "docs/deprecated.md" + - dest: "engine/reference" + src: + - "docs/reference/builder.md" + - "docs/reference/run.md" + - dest: "engine/reference/commandline" + src: + - "docs/reference/commandline/cli.md" + - "docs/reference/commandline/dockerd.md" + + - repo: "https://github.com/docker/docker" + ref: "20.10" + paths: + - dest: "engine/api" + src: + - "docs/api/**" + + - repo: "https://github.com/docker/compose-cli" + ref: "main" + paths: + - dest: "cloud" + src: + - "docs/*.md" + - "!docs/README.md" # readme to make things nice in the compose-cli repo, but meaningless here + - "!docs/architecture.md" # Compose-CLI architecture, unrelated to cloud integration + + - repo: "https://github.com/docker/extensions-sdk" + ref: "main" + paths: + - dest: "desktop/extensions-sdk" + src: + - "docs/**" + + - repo: "https://github.com/distribution/distribution" + ref: "release/2.7" + paths: + - dest: "registry/spec" + src: + - "docs/spec/**" + - "!docs/spec/api.md.tmpl" + - dest: "registry" + src: + - "docs/configuration.md" diff --git a/_plugins/fetch_remote.rb b/_plugins/fetch_remote.rb new file mode 100644 index 0000000000..9c1cdbff00 --- /dev/null +++ b/_plugins/fetch_remote.rb @@ -0,0 +1,64 @@ +require 'archive/zip' +require 'jekyll' +require 'json' +require 'octopress-hooks' +require 'open-uri' +require 'rake' + +module Jekyll + def self.download(url, dest) + uri = URI.parse(url) + result = File.join(dest, File.basename(uri.path)) + Jekyll.logger.info " Downloading #{url}" + IO.copy_stream(URI.open(url), result) + return result + end + + class FetchRemote < Octopress::Hooks::Site + def pre_read(site) + beginning_time = Time.now + Jekyll.logger.info "Starting plugin fetch_remote.rb..." + site.config['fetch-remote'].each do |entry| + Jekyll.logger.info " Repo #{entry['repo']} (#{entry['ref']})" + Dir.mktmpdir do |tmpdir| + tmpfile = Jekyll.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) + Dir.mktmpdir do |ztmpdir| + Jekyll.logger.info " Extracting #{tmpfile}" + Archive::Zip.extract( + tmpfile, + ztmpdir, + :create => true + ) + entry['paths'].each do |path| + Jekyll.logger.info " Copying files to ./#{path['dest']}/" + files = FileList[] + path['src'].each do |src| + if "#{src}".start_with?("!") + files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) + else + files.include(File.join(ztmpdir, "*/#{src}")) + end + end + files.each do |file| + Jekyll.logger.info " #{file.delete_prefix(ztmpdir)}" + end + FileUtils.mkdir_p path['dest'] + FileUtils.cp_r(files, path['dest']) + end + end + end + end + + Jekyll.logger.info " Fixing up URLs in swagger files" + Dir.glob("./engine/api/*.yaml") do |file_name| + Jekyll.logger.info " #{file_name}" + text = File.read(file_name) + replace = text.gsub!("https://docs.docker.com/", "") + File.open(file_name, "w") { |file| file.puts replace } + end + + end_time = Time.now + Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds" + end + end +end diff --git a/_scripts/fetch-upstream-resources.sh b/_scripts/fetch-upstream-resources.sh deleted file mode 100755 index 519b8009d9..0000000000 --- a/_scripts/fetch-upstream-resources.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh - -# Fetches upstream resources from docker/docker and distribution/distribution -# before handing off the site to Jekyll to build -# Relies on the "ENGINE_BRANCH" and "DISTRIBUTION_BRANCH" environment variables, -# which are usually set by the Dockerfile. -: "${ENGINE_BRANCH?No release branch set for docker/docker and docker/cli}" -: "${DISTRIBUTION_BRANCH?No release branch set for distribution/distribution}" -: "${COMPOSE_CLI_BRANCH?No release branch set for docker/compose-cli}" -: "${EXTENSIONS_SDK_BRANCH?No release branch set for docker/extensions-sdk}" - -# Translate branches for use by svn -engine_svn_branch="branches/${ENGINE_BRANCH}" -if [ "${engine_svn_branch}" = "branches/master" ]; then - engine_svn_branch=trunk -fi -distribution_svn_branch="branches/${DISTRIBUTION_BRANCH}" -if [ "${distribution_svn_branch}" = "branches/master" ]; then - distribution_svn_branch=trunk -fi -compose_cli_svn_branch="branches/${COMPOSE_CLI_BRANCH}" -if [ "${compose_cli_svn_branch}" = "branches/main" ]; then - compose_cli_svn_branch=trunk -fi -extensions_sdk_svn_branch="branches/${EXTENSIONS_SDK_BRANCH}" -if [ "${extensions_sdk_svn_branch}" = "branches/main" ]; then - extensions_sdk_svn_branch=trunk -fi - -# Directories to get via SVN. We use this because you can't use git to clone just a portion of a repository -svn co "https://github.com/docker/cli/${engine_svn_branch}/docs/extend" ./engine/extend || (echo "Failed engine/extend download" && exit 1) -svn co "https://github.com/docker/docker/${engine_svn_branch}/docs/api" ./engine/api || (echo "Failed engine/api download" && exit 1) -svn co "https://github.com/docker/compose-cli/${compose_cli_svn_branch}/docs" ./cloud || (echo "Failed compose-cli/docs download" && exit 1) -svn co "https://github.com/docker/extensions-sdk/${extensions_sdk_svn_branch}/docs" ./desktop/extensions-sdk || (echo "Failed extensions-sdk/docs download" && exit 1) -svn co "https://github.com/distribution/distribution/${distribution_svn_branch}/docs/spec" ./registry/spec || (echo "Failed registry/spec download" && exit 1) - -# Fix up URls in swagger files -find ./engine/api -type f -name '*.yaml' | while read i; do sed -i 's#https://docs.docker.com/#/#g' "$i"; done; - -# Cleanup svn directories -find . -name ".svn" -print0 | xargs -0 /bin/rm -rf - -# Get a few one-off files that we use directly from upstream -wget --quiet --directory-prefix=./engine/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/deprecated.md" || (echo "Failed engine/deprecated.md download" && exit 1) -wget --quiet --directory-prefix=./engine/reference/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/builder.md" || (echo "Failed engine/reference/builder.md download" && exit 1) -wget --quiet --directory-prefix=./engine/reference/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/run.md" || (echo "Failed engine/reference/run.md download" && exit 1) -wget --quiet --directory-prefix=./engine/reference/commandline/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/commandline/cli.md" || (echo "Failed engine/reference/commandline/cli.md download" && exit 1) -wget --quiet --directory-prefix=./engine/reference/commandline/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/commandline/dockerd.md" || (echo "Failed engine/reference/commandline/dockerd.md download" && exit 1) -wget --quiet --directory-prefix=./registry/ "https://raw.githubusercontent.com/distribution/distribution/${DISTRIBUTION_BRANCH}/docs/configuration.md" || (echo "Failed registry/configuration.md download" && exit 1) - -# Remove things we don't want in the build -rm -f ./engine/extend/cli_plugins.md # the cli plugins api is not a stable API, and not included in the TOC for that reason. -rm -f ./registry/spec/api.md.tmpl -rm -f ./cloud/README.md # readme to make things nice in the compose-cli repo, but meaningless here -rm -f ./cloud/architecture.md # Compose-CLI architecture, unrelated to cloud integration -rm -rf ./cloud/images diff --git a/docker-bake.hcl b/docker-bake.hcl index aff41a86bf..d058256271 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -11,7 +11,7 @@ target "release" { args = { JEKYLL_ENV = JEKYLL_ENV } - no-cache-filter = ["upstream-resources"] + no-cache-filter = ["generate"] output = ["./_site"] }