Merge pull request #14799 from crazy-max/auto-edit-url-remote

jekyll: automatic edit url generation for remote resources

_config.yml

@@ -90,11 +90,9 @@ sass:
 # limitations in Jekyll cause those to introduce a _severe_ impact on build-time,
 # affecting generation of (e.g.) sitemap.xml and metadata.json, resulting in the
 # total build to take 60 seconds longer to build (!).
-#
-# The list below is for "development" (local builds, and PR previews) builds only,
-# and should be kept minimal to allow for fast builds. Other options should go
-# into _config_production.yml, which is used for production deploys.
 defaults:
+  # Default one for development builds (local and PR previews)
+  # sitemap is disabled here but not for production in _config_production.yml
   - scope:
       path: ""
       type: "pages"
@@ -104,10 +102,22 @@ defaults:
       toc_min: 2
       toc_max: 4
 
+  # Set the correct edit-URL for some local and remote resources. We usually don't create a direct
+  # edit link for these, and instead point to the directory that contains the file.
+  - scope:
+      path: engine/reference
+    values:
+      # FIXME: This edit url is as "best-effort" and doesn't match anything in docker/cli repo for plugins for example. It should be generated autmatically.
+      edit_url: "https://github.com/docker/cli/tree/master/docs/reference"
   - scope:
       path: engine/reference/commandline
     values:
+      edit_url: "https://github.com/docker/cli/tree/master/docs/reference/commandline"
       skip_read_time: true
+  - scope:
+      path: glossary.md
+    values:
+      edit_url: "https://github.com/docker/docker.github.io/blob/master/_data/glossary.yaml"
 
 # Fetch upstream resources (reference documentation) used by _plugins/fetch_remote.rb
 # - repo is the GitHub repository to fetch from
@@ -117,6 +127,7 @@ defaults:
 #   - src is a list of glob source paths within the remote repository
 fetch-remote:
   - repo: "https://github.com/docker/cli"
+    default_branch: "master"
     ref: "20.10"
     paths:
       - dest: "engine/extend"
@@ -135,6 +146,7 @@ fetch-remote:
           - "docs/reference/commandline/dockerd.md"
 
   - repo: "https://github.com/docker/docker"
+    default_branch: "master"
     ref: "20.10"
     paths:
       - dest: "engine/api"
@@ -142,6 +154,7 @@ fetch-remote:
           - "docs/api/**"
 
   - repo: "https://github.com/docker/compose-cli"
+    default_branch: "main"
     ref: "main"
     paths:
       - dest: "cloud"
@@ -151,6 +164,7 @@ fetch-remote:
           - "!docs/architecture.md" # Compose-CLI architecture, unrelated to cloud integration
 
   - repo: "https://github.com/distribution/distribution"
+    default_branch: "main"
     ref: "release/2.7"
     paths:
       - dest: "registry/spec"

_config_production.yml

@@ -54,66 +54,6 @@ defaults:
       toc_min: 2
       toc_max: 4
 
-  # Set the correct edit-URL for upstream resources. We usually don't create a direct
-  # edit link for these, and instead point to the directory that contains the file.
-  - scope:
-      path: cloud/aci-compose-features.md
-    values:
-      edit_url: "https://github.com/docker/compose-cli/tree/master/docs/"
-  - scope:
-      path: cloud/aci-container-features.md
-    values:
-      edit_url: "https://github.com/docker/compose-cli/tree/master/docs/"
-  - scope:
-      path: cloud/ecs-architecture.md
-    values:
-      edit_url: "https://github.com/docker/compose-cli/tree/master/docs/"
-  - scope:
-      path: cloud/ecs-compose-features.md
-    values:
-      edit_url: "https://github.com/docker/compose-cli/tree/master/docs/"
-  - scope:
-      path: cloud/ecs-compose-examples.md
-    values:
-      edit_url: "https://github.com/docker/compose-cli/tree/master/docs/"
-  - scope:
-      path: engine/deprecated.md
-    values:
-      edit_url: "https://github.com/docker/cli/tree/master/docs/"
-  - scope:
-      path: engine/extend.md
-    values:
-      edit_url: "https://github.com/docker/cli/tree/master/docs/extend"
-  - scope:
-      path: engine/extend
-    values:
-      edit_url: "https://github.com/docker/cli/tree/master/docs/extend"
-  - scope:
-      path: engine/reference
-    values:
-      edit_url: "https://github.com/docker/cli/tree/master/docs/reference"
-  - scope:
-      path: engine/reference/commandline
-    values:
-      edit_url: "https://github.com/docker/cli/tree/master/docs/reference/commandline"
-      skip_read_time: true
-  - scope:
-      path: glossary.md
-    values:
-      edit_url: "https://github.com/docker/docker.github.io/blob/master/_data/glossary.yaml"
-  - scope:
-      path: notary/reference
-    values:
-      edit_url: "https://github.com/theupdateframework/notary/tree/master/docs/reference"
-  - scope:
-      path: registry/configuration
-    values:
-      edit_url: "https://github.com/docker/distribution/tree/master/docs"
-  - scope:
-      path: registry/spec
-    values:
-      edit_url: "https://github.com/docker/distribution/tree/master/docs/spec"
-
   # Hide Enterprise content from sitemap. These directories still contains stubs
   # that are needed to redirect old URLs
   - scope:

_layouts/docs.html

@@ -4,6 +4,11 @@
     {%- assign edit_url = page.edit_url -%}
 {%- else -%}
     {%- assign edit_url = "https://github.com/docker/docker.github.io/edit/master/" | append: page.path -%}
+{%- endif -%}
+{%- if page.issue_url -%}
+    {%- assign issue_url = page.issue_url -%}
+{%- else -%}
+    {%- assign issue_url = "https://github.com/docker/docker.github.io/issues/new?body=File: [" | append: page.path | append: "](https://docs.docker.com" | append: page.url | append: ")" -%}
 {%- endif -%} -->
 <html lang="en">
 {%- include head.html -%}
@@ -63,7 +68,9 @@
                                         {%- if edit_url != "" -%}
                                         <li><a href="{{ edit_url }}"><i class="fa fa-pencil-square-o" aria-hidden="true"></i> Edit this page</a></li>
                                         {%- endif -%}
-                                        <li><a href="https://github.com/docker/docker.github.io/issues/new?body=File: [{{ page.path }}](https://docs.docker.com{{ page.url }})" class="nomunge"><i class="fa fa-check" aria-hidden="true"></i> Request docs changes</a></li>
+                                        {%- if issue_url != "" -%}
+                                        <li><a href="{{ issue_url }}" class="nomunge"><i class="fa fa-check" aria-hidden="true"></i> Request docs changes</a></li>
+                                        {%- endif -%}
                                         <li>{%- include theme-switch.html -%}</li>
                                     </ul>
                                 </div>

_plugins/fetch_remote.rb

@@ -9,28 +9,47 @@ module Jekyll
   def self.download(url, dest)
     uri = URI.parse(url)
     result = File.join(dest, File.basename(uri.path))
-    Jekyll.logger.info "    Downloading #{url}"
+    puts "    Downloading #{url}"
     IO.copy_stream(URI.open(url), result)
     return result
   end
 
+  def self.copy(src, dest)
+    if (tmp = Array.try_convert(src))
+      tmp.each do |s|
+        s = File.path(s)
+        yield s, File.join(dest, File.basename(s))
+      end
+    else
+      src = File.path(src)
+      if File.directory?(dest)
+        yield src, File.join(dest, File.basename(src))
+      else
+        yield src, File.path(dest)
+      end
+    end
+  end
+
   class FetchRemote < Octopress::Hooks::Site
     def pre_read(site)
       beginning_time = Time.now
-      Jekyll.logger.info "Starting plugin fetch_remote.rb..."
+      puts "Starting plugin fetch_remote.rb..."
       site.config['fetch-remote'].each do |entry|
-        Jekyll.logger.info "  Repo #{entry['repo']} (#{entry['ref']})"
+        puts "  Repo #{entry['repo']} (#{entry['ref']})"
         Dir.mktmpdir do |tmpdir|
           tmpfile = Jekyll.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir)
           Dir.mktmpdir do |ztmpdir|
-            Jekyll.logger.info "    Extracting #{tmpfile}"
+            puts "    Extracting #{tmpfile}"
             Archive::Zip.extract(
               tmpfile,
               ztmpdir,
               :create => true
             )
             entry['paths'].each do |path|
-              Jekyll.logger.info "    Copying files to ./#{path['dest']}/"
+              FileUtils.mkdir_p path['dest']
+              puts "    Copying files"
+
+              # prepare file list to be copied
               files = FileList[]
               path['src'].each do |src|
                 if "#{src}".start_with?("!")
@@ -39,18 +58,40 @@ module Jekyll
                   files.include(File.join(ztmpdir, "*/#{src}"))
                 end
               end
+
               files.each do |file|
-                Jekyll.logger.info "      #{file.delete_prefix(ztmpdir)}"
+                Jekyll.copy(file, path['dest']) do |s, d|
+                  s = File.realpath(s)
+                  # traverse source directory
+                  FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent|
+                    file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/")
+                    destent = FileUtils::Entry_.new(d, ent.rel, false)
+                    puts "      #{file_clean} => #{destent.path}"
+                    ent.copy destent.path
+
+                    next unless File.file?(ent.path) && File.extname(ent.path) == ".md"
+                    # set edit and issue url and remote info for markdown files in site config defaults
+                    edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}"
+                    issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](https://docs.docker.com/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)"
+                    puts "        edit_url:  #{edit_url}"
+                    puts "        issue_url: #{issue_url}"
+                    site.config['defaults'] << {
+                      "scope" => { "path" => destent.path },
+                      "values" => {
+                        "edit_url" => edit_url,
+                        "issue_url" => issue_url
+                      },
+                    }
+                  end, proc do |_| end)
+                end
               end
-              FileUtils.mkdir_p path['dest']
-              FileUtils.cp_r(files, path['dest'])
             end
           end
         end
       end
 
       end_time = Time.now
-      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+      puts "done in #{(end_time - beginning_time)} seconds"
     end
   end
 end