jekyll: create plugin to fetch remote resources

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

Dockerfile

@@ -12,15 +12,10 @@ ARG BUNDLER_VERSION=2.3.13
 ARG JEKYLL_ENV=development
 ARG DOMAIN=docs.docker.com
 
-ARG ENGINE_BRANCH="20.10"
-ARG DISTRIBUTION_BRANCH="release/2.7"
-ARG COMPOSE_CLI_BRANCH="main"
-ARG EXTENSIONS_SDK_BRANCH="main"
-
 # Base stage for building
 FROM ruby:${RUBY_VERSION}-alpine AS base
 WORKDIR /src
-RUN apk add --no-cache bash build-base git subversion wget
+RUN apk add --no-cache bash build-base git
 
 # Gem stage will install bundler used as dependency manager
 # for our dependencies in Gemfile for Jekyll
@@ -45,27 +40,14 @@ RUN bundle update \
 FROM scratch AS vendor
 COPY --from=vendored /out /
 
-# Fetch upstream resources (reference documentation)
-# Only add the files that are needed to build these reference docs, so that these
-# docs are only rebuilt if changes were made to ENGINE_BRANCH or DISTRIBUTION_BRANCH.
-FROM base AS upstream-resources
-WORKDIR /out
-COPY ./_scripts/fetch-upstream-resources.sh ./_scripts/
-ARG ENGINE_BRANCH
-ARG DISTRIBUTION_BRANCH
-ARG COMPOSE_CLI_BRANCH
-ARG EXTENSIONS_SDK_BRANCH
-RUN ./_scripts/fetch-upstream-resources.sh .
-
 # Build the static HTML for the current docs.
 # After building with jekyll, fix up some links
 FROM gem AS generate
 ARG JEKYLL_ENV
 ARG DOMAIN
 ENV TARGET=/out
-COPY . .
+RUN --mount=type=bind,target=.,rw \
-COPY --from=upstream-resources /out .
+  --mount=type=cache,target=/src/.jekyll-cache <<EOT
-RUN --mount=type=cache,target=/src/.jekyll-cache <<EOT
 set -eu
 
 # substitute the "{site.latest_engine_api_version}" in the title for the latest

Gemfile

@@ -2,8 +2,13 @@ source 'https://rubygems.org'
 
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
 gem 'wdm', '>= 0.1.0' if Gem.win_platform?
+
 gem 'jekyll'
 gem 'jekyll-redirect-from'
 gem 'jekyll-relative-links'
 gem 'jekyll-sitemap'
+
+gem 'archive-zip'
 gem 'html-proofer'
+gem 'octopress-hooks'
+gem 'rake'

Gemfile.lock

@@ -3,6 +3,8 @@ GEM
   specs:
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
+    archive-zip (0.12.0)
+      io-like (~> 0.3.0)
     colorator (1.1.0)
     concurrent-ruby (1.1.10)
     em-websocket (0.5.3)
@@ -24,6 +26,7 @@ GEM
     http_parser.rb (0.8.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    io-like (0.3.1)
     jekyll (4.2.2)
       addressable (~> 2.4)
       colorator (~> 1.0)
@@ -58,14 +61,17 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.4.0)
-    nokogiri (1.13.5-x86_64-linux)
+    nokogiri (1.13.6-x86_64-linux)
       racc (~> 1.4)
+    octopress-hooks (2.6.2)
+      jekyll (>= 2.0)
     parallel (1.22.1)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
     racc (1.6.0)
     rainbow (3.1.1)
+    rake (13.0.6)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -87,11 +93,14 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  archive-zip
   html-proofer
   jekyll
   jekyll-redirect-from
   jekyll-relative-links
   jekyll-sitemap
+  octopress-hooks
+  rake
   tzinfo-data
 
 BUNDLED WITH

_config.yml

@@ -104,3 +104,62 @@ defaults:
       path: engine/reference/commandline
     values:
       skip_read_time: true
+
+# Fetch upstream resources (reference documentation) used by _plugins/fetch_remote.rb
+# - repo is the GitHub repository to fetch from
+# - ref the Git reference
+# - paths is a list to the resources within the remote repository
+#   - dest is the destination path within the working tree
+#   - src is a list of glob source paths within the remote repository
+fetch-remote:
+  - repo: "https://github.com/docker/cli"
+    ref: "20.10"
+    paths:
+      - dest: "engine/extend"
+        src:
+          - "docs/extend/**"
+      - dest: "engine"
+        src:
+          - "docs/deprecated.md"
+      - dest: "engine/reference"
+        src:
+          - "docs/reference/builder.md"
+          - "docs/reference/run.md"
+      - dest: "engine/reference/commandline"
+        src:
+          - "docs/reference/commandline/cli.md"
+          - "docs/reference/commandline/dockerd.md"
+
+  - repo: "https://github.com/docker/docker"
+    ref: "20.10"
+    paths:
+      - dest: "engine/api"
+        src:
+          - "docs/api/**"
+
+  - repo: "https://github.com/docker/compose-cli"
+    ref: "main"
+    paths:
+      - dest: "cloud"
+        src:
+          - "docs/*.md"
+          - "!docs/README.md" # readme to make things nice in the compose-cli repo, but meaningless here
+          - "!docs/architecture.md" # Compose-CLI architecture, unrelated to cloud integration
+
+  - repo: "https://github.com/docker/extensions-sdk"
+    ref: "main"
+    paths:
+      - dest: "desktop/extensions-sdk"
+        src:
+          - "docs/**"
+
+  - repo: "https://github.com/distribution/distribution"
+    ref: "release/2.7"
+    paths:
+      - dest: "registry/spec"
+        src:
+          - "docs/spec/**"
+          - "!docs/spec/api.md.tmpl"
+      - dest: "registry"
+        src:
+          - "docs/configuration.md"

_plugins/fetch_remote.rb

@@ -0,0 +1,64 @@
+require 'archive/zip'
+require 'jekyll'
+require 'json'
+require 'octopress-hooks'
+require 'open-uri'
+require 'rake'
+
+module Jekyll
+  def self.download(url, dest)
+    uri = URI.parse(url)
+    result = File.join(dest, File.basename(uri.path))
+    Jekyll.logger.info "    Downloading #{url}"
+    IO.copy_stream(URI.open(url), result)
+    return result
+  end
+
+  class FetchRemote < Octopress::Hooks::Site
+    def pre_read(site)
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin fetch_remote.rb..."
+      site.config['fetch-remote'].each do |entry|
+        Jekyll.logger.info "  Repo #{entry['repo']} (#{entry['ref']})"
+        Dir.mktmpdir do |tmpdir|
+          tmpfile = Jekyll.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir)
+          Dir.mktmpdir do |ztmpdir|
+            Jekyll.logger.info "    Extracting #{tmpfile}"
+            Archive::Zip.extract(
+              tmpfile,
+              ztmpdir,
+              :create => true
+            )
+            entry['paths'].each do |path|
+              Jekyll.logger.info "    Copying files to ./#{path['dest']}/"
+              files = FileList[]
+              path['src'].each do |src|
+                if "#{src}".start_with?("!")
+                  files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!")))
+                else
+                  files.include(File.join(ztmpdir, "*/#{src}"))
+                end
+              end
+              files.each do |file|
+                Jekyll.logger.info "      #{file.delete_prefix(ztmpdir)}"
+              end
+              FileUtils.mkdir_p path['dest']
+              FileUtils.cp_r(files, path['dest'])
+            end
+          end
+        end
+      end
+
+      Jekyll.logger.info "  Fixing up URLs in swagger files"
+      Dir.glob("./engine/api/*.yaml") do |file_name|
+        Jekyll.logger.info "    #{file_name}"
+        text = File.read(file_name)
+        replace = text.gsub!("https://docs.docker.com/", "")
+        File.open(file_name, "w") { |file| file.puts replace }
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+end

_scripts/fetch-upstream-resources.sh

@@ -1,56 +0,0 @@
-#!/bin/sh
-
-# Fetches upstream resources from docker/docker and distribution/distribution
-# before handing off the site to Jekyll to build
-# Relies on the "ENGINE_BRANCH" and "DISTRIBUTION_BRANCH" environment variables,
-# which are usually set by the Dockerfile.
-: "${ENGINE_BRANCH?No release branch set for docker/docker and docker/cli}"
-: "${DISTRIBUTION_BRANCH?No release branch set for distribution/distribution}"
-: "${COMPOSE_CLI_BRANCH?No release branch set for docker/compose-cli}"
-: "${EXTENSIONS_SDK_BRANCH?No release branch set for docker/extensions-sdk}"
-
-# Translate branches for use by svn
-engine_svn_branch="branches/${ENGINE_BRANCH}"
-if [ "${engine_svn_branch}" = "branches/master" ]; then
-	engine_svn_branch=trunk
-fi
-distribution_svn_branch="branches/${DISTRIBUTION_BRANCH}"
-if [ "${distribution_svn_branch}" = "branches/master" ]; then
-	distribution_svn_branch=trunk
-fi
-compose_cli_svn_branch="branches/${COMPOSE_CLI_BRANCH}"
-if [ "${compose_cli_svn_branch}" = "branches/main" ]; then
-	compose_cli_svn_branch=trunk
-fi
-extensions_sdk_svn_branch="branches/${EXTENSIONS_SDK_BRANCH}"
-if [ "${extensions_sdk_svn_branch}" = "branches/main" ]; then
-	extensions_sdk_svn_branch=trunk
-fi
-
-# Directories to get via SVN. We use this because you can't use git to clone just a portion of a repository
-svn co "https://github.com/docker/cli/${engine_svn_branch}/docs/extend"                    ./engine/extend || (echo "Failed engine/extend download" && exit 1)
-svn co "https://github.com/docker/docker/${engine_svn_branch}/docs/api"                    ./engine/api    || (echo "Failed engine/api download" && exit 1)
-svn co "https://github.com/docker/compose-cli/${compose_cli_svn_branch}/docs"              ./cloud         || (echo "Failed compose-cli/docs download" && exit 1)
-svn co "https://github.com/docker/extensions-sdk/${extensions_sdk_svn_branch}/docs"              		   ./desktop/extensions-sdk         || (echo "Failed extensions-sdk/docs download" && exit 1)
-svn co "https://github.com/distribution/distribution/${distribution_svn_branch}/docs/spec" ./registry/spec || (echo "Failed registry/spec download" && exit 1)
-
-# Fix up URls in swagger files
-find ./engine/api -type f -name '*.yaml' | while read i; do sed -i 's#https://docs.docker.com/#/#g' "$i"; done;
-
-# Cleanup svn directories
-find . -name ".svn" -print0 | xargs -0 /bin/rm -rf
-
-# Get a few one-off files that we use directly from upstream
-wget --quiet --directory-prefix=./engine/                       "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/deprecated.md"                          || (echo "Failed engine/deprecated.md download" && exit 1)
-wget --quiet --directory-prefix=./engine/reference/             "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/builder.md"                   || (echo "Failed engine/reference/builder.md download" && exit 1)
-wget --quiet --directory-prefix=./engine/reference/             "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/run.md"                       || (echo "Failed engine/reference/run.md download" && exit 1)
-wget --quiet --directory-prefix=./engine/reference/commandline/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/commandline/cli.md"           || (echo "Failed engine/reference/commandline/cli.md download" && exit 1)
-wget --quiet --directory-prefix=./engine/reference/commandline/ "https://raw.githubusercontent.com/docker/cli/${ENGINE_BRANCH}/docs/reference/commandline/dockerd.md"       || (echo "Failed engine/reference/commandline/dockerd.md download" && exit 1)
-wget --quiet --directory-prefix=./registry/                     "https://raw.githubusercontent.com/distribution/distribution/${DISTRIBUTION_BRANCH}/docs/configuration.md"  || (echo "Failed registry/configuration.md download" && exit 1)
-
-# Remove things we don't want in the build
-rm -f ./engine/extend/cli_plugins.md # the cli plugins api is not a stable API, and not included in the TOC for that reason.
-rm -f ./registry/spec/api.md.tmpl
-rm -f ./cloud/README.md # readme to make things nice in the compose-cli repo, but meaningless here
-rm -f ./cloud/architecture.md # Compose-CLI architecture, unrelated to cloud integration
-rm -rf ./cloud/images

docker-bake.hcl

@@ -11,7 +11,7 @@ target "release" {
   args = {
     JEKYLL_ENV = JEKYLL_ENV
   }
-  no-cache-filter = ["upstream-resources"]
+  no-cache-filter = ["generate"]
   output = ["./_site"]
 }