Merge pull request #15250 from crazy-max/enable-metadata-and-sitemap

jekyll: enable sitemap and local search for dev environment

.github/workflows/publish.yml

@@ -18,17 +18,20 @@ jobs:
           JEKYLL_ENV=development
           DOCS_AWS_REGION=us-east-1
           if [ "${{ github.ref }}" = "refs/heads/master" ]; then
+            DOCS_URL="https://docs-stage.docker.com"
             DOCS_S3_BUCKET="docs.docker.com-stage-us-east-1"
             DOCS_S3_CONFIG="_website-config-docs-stage.json"
             DOCS_LAMBDA_FUNCTION_CACHE="arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator"
             DOCS_SLACK_MSG="Successfully promoted docs-stage from master. https://docs-stage.docker.com/"
           elif [ "${{ github.ref }}" = "refs/heads/published" ]; then
             JEKYLL_ENV=production
+            DOCS_URL="https://docs.docker.com"
             DOCS_S3_BUCKET="docs.docker.com-us-east-1"
             DOCS_S3_CONFIG="_website-config-docs.json"
             DOCS_LAMBDA_FUNCTION_CACHE="arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator"
             DOCS_SLACK_MSG="Successfully published docs. https://docs.docker.com/"
           elif [ "${{ github.ref }}" = "refs/heads/lab" ]; then
+            DOCS_URL="https://docs-lab.docker.com"
             DOCS_S3_BUCKET=""
             DOCS_LAMBDA_FUNCTION_CACHE=""
             DOCS_S3_CONFIG="_website-config-docs-lab.json"
@@ -37,6 +40,7 @@ jobs:
             exit 1
           fi
           echo "JEKYLL_ENV=$JEKYLL_ENV" >> $GITHUB_ENV
+          echo "DOCS_URL=$DOCS_URL" >> $GITHUB_ENV
           echo "DOCS_AWS_REGION=$DOCS_AWS_REGION" >> $GITHUB_ENV
           echo "DOCS_S3_BUCKET=$DOCS_S3_BUCKET" >> $GITHUB_ENV
           echo "DOCS_S3_CONFIG=$DOCS_S3_CONFIG" >> $GITHUB_ENV

Dockerfile

@@ -10,7 +10,7 @@ ARG RUBY_VERSION=2.7.6
 ARG BUNDLER_VERSION=2.3.13
 
 ARG JEKYLL_ENV=development
-ARG DOMAIN=docs.docker.com
+ARG DOCS_URL=http://localhost:4000
 
 # Base stage for building
 FROM ruby:${RUBY_VERSION}-alpine AS base
@@ -44,28 +44,14 @@ COPY --from=vendored /out /
 # After building with jekyll, fix up some links
 FROM gem AS generate
 ARG JEKYLL_ENV
-ARG DOMAIN
+ARG DOCS_URL
 ENV TARGET=/out
 RUN --mount=type=bind,target=.,rw \
     --mount=type=cache,target=/src/.jekyll-cache <<EOT
   set -eu
-if [ "${JEKYLL_ENV}" = "production" ]; then
+  CONFIG_FILES=_config.yml$([ "$JEKYLL_ENV" = "production" ] && echo ",_config_production.yml" || true)
-  (
   set -x
-    bundle exec jekyll build --profile -d ${TARGET} --config _config.yml,_config_production.yml
+  bundle exec jekyll build --profile -d ${TARGET} --config ${CONFIG_FILES}
-    sed -i "s#<loc>/#<loc>https://${DOMAIN}/#" "${TARGET}/sitemap.xml"
-  )
-else
-  (
-    set -x
-    bundle exec jekyll build --trace --profile -d ${TARGET}
-    mkdir -p ${TARGET}/js
-    echo '[]' > ${TARGET}/js/metadata.json
-  )
-fi
-find ${TARGET} -type f -name '*.html' | while read i; do
-  sed -i "s#\(<a[^>]* href=\"\)https://${DOMAIN}/#\1/#g" "$i"
-done
 EOT
 
 # htmlproofer checks for broken links

_config.yml

@@ -30,7 +30,6 @@ exclude:
   - datacenter
   - docker-bake.hcl
   - ee
-  - js/metadata.json
   - LICENSE
   - Makefile
   - README.md
@@ -67,12 +66,15 @@ compose_switch_version: "1.0.4"
 # which we show a badge (currently: API v1.40, or "Docker 19.03").
 min_api_threshold: 1.40
 
+# Enable search autocompletion (requires metadata.json to be generated)
+local_search: true
+
 # List of plugins to enable for local development builds. Mostly the same as
-# for production, but without the "jekyll-sitemap" plugin, which is not needed
+# for production.
-# for previewing, and excluding saves some time to build
 plugins:
   - jekyll-redirect-from
   - jekyll-relative-links
+  - jekyll-sitemap
 
 # Assets
 #
@@ -87,10 +89,7 @@ sass:
 # Set default options / metadata for some paths.
 #
 # Setting options here prevents having to repeat the same option in front-matter
-# on every page. Avoid using wildcards, such as "path: engine/api/v1.*", as
+# on every page.
-# limitations in Jekyll cause those to introduce a _severe_ impact on build-time,
-# affecting generation of (e.g.) sitemap.xml and metadata.json, resulting in the
-# total build to take 60 seconds longer to build (!).
 defaults:
   # Default one for development builds (local and PR previews)
   # sitemap is disabled here but not for production in _config_production.yml
@@ -99,7 +98,6 @@ defaults:
       type: "pages"
     values:
       layout: docs
-      sitemap: false
       toc_min: 2
       toc_max: 4
 

_config_production.yml

@@ -17,14 +17,6 @@ exclude:
 google_analytics: GTM-WL2QLG5
 polldaddy_id:     8453675
 
-# Enable search autocompletion (requires metadata.json to be generated)
-local_search: true
-
-plugins:
-  - jekyll-redirect-from
-  - jekyll-relative-links
-  - jekyll-sitemap
-
 # Assets
 #
 # For production/deploy, we build css with the "compressed" format, to produce
@@ -39,10 +31,7 @@ collections:
 # Set default options / metadata for some paths.
 #
 # Setting options here prevents having to repeat the same option in front-matter
-# on every page. Avoid using wildcards, such as "path: engine/api/v1.*", as
+# on every page.
-# limitations in Jekyll cause those to introduce a _severe_ impact on build-time,
-# affecting generation of (e.g.) sitemap.xml and metadata.json, resulting in the
-# total build to take 60 seconds longer to build (!).
 #
 # The list below is used for *production* deploys, and overrides the one defined
 # in "_config.yml", which is used for local builds and pull-request previews.

_plugins/fix_swagger.rb

@@ -0,0 +1,25 @@
+require 'jekyll'
+require 'octopress-hooks'
+
+require_relative 'util.rb'
+
+module Jekyll
+  class FetchRemote < Octopress::Hooks::Site
+    def post_read(site)
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin fix_swagger.rb..."
+
+      files = Dir.glob(%w[./docker-hub/api/*.yaml ./engine/api/*.yaml])
+      Jekyll.logger.info "  Fixing up #{files.size} swagger file(s)..."
+      files.each do |f|
+        Jekyll.logger.info "    #{f}"
+        text = File.read(f)
+        replace = text.gsub(get_docs_url, "")
+        File.open(f, "w") { |f2| f2.puts replace }
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+end

_plugins/fix_url.rb

@@ -0,0 +1,29 @@
+require 'jekyll'
+require 'octopress-hooks'
+
+require_relative 'util.rb'
+
+module Jekyll
+  class FetchRemote < Octopress::Hooks::Site
+    def post_write(site)
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin fix_url.rb..."
+
+      # TODO: use dynamic URL from util.get_docs_url instead of hardcoded one
+      #   but needs to remove first all absolute URLs in our code base.
+      docs_url = "https://docs.docker.com"
+
+      dest = site.config['destination'] || '_site'
+      files = Dir.glob("#{dest}/**/*.html")
+      Jekyll.logger.info "  Fixing up URLs in #{files.size} html file(s) to be relative"
+      files.each do|f|
+        text = File.read(f)
+        replace = text.gsub(/(<a[^>]* href=\")#{docs_url}/, '\1')
+        File.open(f, "w") { |f2| f2.puts replace }
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+end

_plugins/fix_urls.rb

@@ -1,22 +0,0 @@
-require 'jekyll'
-require 'octopress-hooks'
-
-module Jekyll
-  class FetchRemote < Octopress::Hooks::Site
-    def post_read(site)
-      beginning_time = Time.now
-      Jekyll.logger.info "Starting plugin fix_urls.rb..."
-
-      Jekyll.logger.info "  Fixing up URLs in swagger files"
-      Dir.glob(%w[./docker-hub/api/*.yaml ./engine/api/*.yaml]) do |file_name|
-        Jekyll.logger.info "    #{file_name}"
-        text = File.read(file_name)
-        replace = text.gsub("https://docs.docker.com", "")
-        File.open(file_name, "w") { |file| file.puts replace }
-      end
-
-      end_time = Time.now
-      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
-    end
-  end
-end

_plugins/update_sitemap.rb

@@ -0,0 +1,18 @@
+require_relative 'util.rb'
+
+Jekyll::Hooks.register :site, :post_write do |site|
+  beginning_time = Time.now
+  Jekyll.logger.info "Starting plugin update_sitemap.rb..."
+
+  sitemap_path = File.join(site.dest, 'sitemap.xml')
+
+  if File.exist?(sitemap_path)
+    sitemap_file = File.read(sitemap_path)
+    replace = sitemap_file.gsub!("<loc>/", "<loc>#{get_docs_url}/")
+    Jekyll.logger.info "  Replacing '<loc>/' with '<loc>#{get_docs_url}/' in #{sitemap_path}"
+    File.open(sitemap_path, "w") { |file| file.puts replace }
+  end
+
+  end_time = Time.now
+  Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+end

_plugins/util.rb

@@ -0,0 +1,5 @@
+def get_docs_url
+  # DEPLOY_URL is from Netlify for preview
+  # https://docs.netlify.com/configure-builds/environment-variables/#deploy-urls-and-metadata
+  ENV['DEPLOY_URL'] || ENV['DOCS_URL'] || 'https://docs.docker.com'
+end

docker-bake.hcl

@@ -1,6 +1,9 @@
 variable "JEKYLL_ENV" {
   default = "development"
 }
+variable "DOCS_URL" {
+  default = "http://localhost:4000"
+}
 variable "DOCS_SITE_DIR" {
   default = "_site"
 }
@@ -8,6 +11,7 @@ variable "DOCS_SITE_DIR" {
 target "_common" {
   args = {
     JEKYLL_ENV = JEKYLL_ENV
+    DOCS_URL = DOCS_URL
   }
   no-cache-filter = ["generate"]
 }

docker-compose.yml

@@ -10,6 +10,7 @@ services:
     build:
       args:
         - JEKYLL_ENV
+        - DOCS_URL
       context: .
     image: docs/docstage
     ports:

sitemap.xsl

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="2.0"
+                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:sitemap="http://www.sitemaps.org/schemas/sitemap/0.9"
+                xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
+                xmlns:html="http://www.w3.org/TR/REC-html40">
+  <xsl:output version="1.0" method="html" encoding="UTF-8" />
+  <xsl:template match="/">
+    <html xmlns="http://www.w3.org/1999/xhtml">
+      <head>
+        <title>Docker Docs XML Sitemap</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <style type="text/css">
+          html, body {
+            font-family: Arial, sans-serif;
+            font-size: 16px;
+            line-height: 1.0;
+          }
+          a {
+            color: #000;
+          }
+          .intro {
+            background-color: #CFEBF7;
+            border: 1px #2580B2 solid;
+            padding: 5px 13px 5px 13px;
+            margin: 10px;
+            width: 800px;
+          }
+          .intro p {
+            line-height: 0.5;
+          }
+          .list td, .list th {
+            padding: 5px;
+            font-size: 13px;
+            line-height: 1.5;
+          }
+          .list th {
+            text-align: left;
+          }
+          tr.high {
+            background-color: whitesmoke;
+          }
+        </style>
+      </head>
+      <body>
+        <h1>Docker Docs XML Sitemap</h1>
+        <div class="intro">
+          <p>This is an XML Sitemap which is supposed to be processed by search engines like <a href="https://www.google.com">Google</a>, <a href="https://www.bing.com">Bing</a>, ...</p>
+          <p>You can find more information about XML sitemaps on <a href="https://sitemaps.org/">sitemaps.org</a> and Google's <a href="https://code.google.com/archive/p/sitemap-generators/wikis/SitemapGenerators.wiki">list of sitemap programs</a>.</p>
+          <p>This sitemap contains <strong><xsl:value-of select="count(sitemap:urlset/sitemap:url)"/></strong> URLs.</p>
+        </div>
+        <div id="content">
+          <table class="list" cellpadding="5">
+            <tr style="border-bottom:1px black solid;">
+              <th>Location</th>
+              <th>Last Modification</th>
+            </tr>
+            <xsl:for-each select="sitemap:urlset/sitemap:url">
+              <tr>
+                <xsl:if test="position() mod 2 != 1">
+                  <xsl:attribute  name="class">high</xsl:attribute>
+                </xsl:if>
+                <td>
+                  <xsl:variable name="itemURL">
+                    <xsl:value-of select="sitemap:loc"/>
+                  </xsl:variable>
+                  <a href="{$itemURL}">
+                    <xsl:value-of select="sitemap:loc"/>
+                  </a>
+                </td>
+                <td>
+                  <xsl:value-of select="concat(substring(sitemap:lastmod,0,11),concat(' ', substring(sitemap:lastmod,12,5)))"/>
+                </td>
+              </tr>
+            </xsl:for-each>
+          </table>
+        </div>
+      </body>
+    </html>
+  </xsl:template>
+</xsl:stylesheet>