jekyll: use git to fetch remote resources

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

Dockerfile

@@ -47,6 +47,7 @@ ARG JEKYLL_ENV
 ARG DOCS_URL
 ENV TARGET=/out
 RUN --mount=type=bind,target=.,rw \
+    --mount=type=cache,target=/tmp/docker-docs-clone \
     --mount=type=cache,target=/src/.jekyll-cache <<EOT
   set -eu
   CONFIG_FILES=_config.yml$([ "$JEKYLL_ENV" = "production" ] && echo ",_config_production.yml" || true)

Gemfile

@@ -16,8 +16,8 @@ end
 #  more info: https://github.com/docker/docs/issues/14788
 gem 'rouge', '3.27.0'
 
-gem 'archive-zip', '0.12.0'
 gem 'front_matter_parser', '1.0.1'
+gem 'git', '1.12.0'
 gem 'html-proofer', '3.19.4'
 gem 'mdl', '0.11.0'
 gem 'octopress-hooks', '2.6.2'

Gemfile.lock

@@ -1,11 +1,9 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
+    addressable (2.8.1)
-      public_suffix (>= 2.0.2, < 5.0)
+      public_suffix (>= 2.0.2, < 6.0)
-    archive-zip (0.12.0)
+    chef-utils (18.0.161)
-      io-like (~> 0.3.0)
-    chef-utils (17.10.0)
       concurrent-ruby
     colorator (1.1.0)
     concurrent-ruby (1.1.10)
@@ -18,6 +16,9 @@ GEM
     ffi (1.15.5)
     forwardable-extended (2.6.0)
     front_matter_parser (1.0.1)
+    git (1.12.0)
+      addressable (~> 2.8)
+      rchardet (~> 1.8)
     html-proofer (3.19.4)
       addressable (~> 2.3)
       mercenary (~> 0.3)
@@ -29,7 +30,6 @@ GEM
     http_parser.rb (0.8.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    io-like (0.3.1)
     jekyll (4.2.2)
       addressable (~> 2.4)
       colorator (~> 1.0)
@@ -92,13 +92,14 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     posix-spawn (0.3.15)
-    public_suffix (4.0.7)
+    public_suffix (5.0.0)
     racc (1.6.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    rb-fsevent (0.11.1)
+    rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    rchardet (1.8.0)
     rexml (3.2.5)
     rouge (3.27.0)
     safe_yaml (1.0.5)
@@ -118,8 +119,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  archive-zip (= 0.12.0)
   front_matter_parser (= 1.0.1)
+  git (= 1.12.0)
   html-proofer (= 3.19.4)
   jekyll (= 4.2.2)
   jekyll-last-modified-at

_plugins/fetch_remote.rb

@@ -1,9 +1,8 @@
-require 'archive/zip'
 require 'front_matter_parser'
+require 'git'
 require 'jekyll'
 require 'json'
 require 'octopress-hooks'
-require 'open-uri'
 require 'rake'
 
 require_relative 'util.rb'
@@ -12,14 +11,6 @@ module Jekyll
   class FetchRemote < Octopress::Hooks::Site
     priority :highest
 
-    def self.download(url, dest)
-      uri = URI.parse(url)
-      result = File.join(dest, File.basename(uri.path))
-      puts "    Downloading #{url}"
-      IO.copy_stream(URI.open(url), result)
-      return result
-    end
-
     def self.copy(src, dest)
       if (tmp = Array.try_convert(src))
         tmp.each do |s|
@@ -50,75 +41,86 @@ module Jekyll
       beginning_time = Time.now
       puts "Starting plugin fetch_remote.rb..."
       site.config['fetch-remote'].each do |entry|
-        puts "  Repo #{entry['repo']} (#{entry['ref']})"
+        puts "  Repo #{entry['repo']}"
-        Dir.mktmpdir do |tmpdir|
-          tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir)
-          Dir.mktmpdir do |ztmpdir|
-            puts "    Extracting #{tmpfile}"
-            Archive::Zip.extract(
-              tmpfile,
-              ztmpdir,
-              :create => true
-            )
-            entry['paths'].each do |path|
-              if File.extname(path['dest']) != ""
-                if path['src'].size > 1
-                  raise "Cannot use file destination #{path['dest']} with multiple sources"
-                end
-                FileUtils.mkdir_p File.dirname(path['dest'])
-              else
-                FileUtils.mkdir_p path['dest']
-              end
 
-              puts "    Copying files"
+        gituri = Git::URL.parse(entry['repo'])
+        clonedir = "#{Dir.tmpdir}/docker-docs-clone#{gituri.path}"
+        if Dir.exist?(clonedir)
+          puts "    Opening #{clonedir}"
+          begin
+            git = Git.open(clonedir)
+            git.chdir do
+              puts "    Fetching #{entry['ref']}"
+              git.checkout(entry['ref'])
+              git.fetch
+            end
+          rescue => e
+            FileUtils.rm_rf(clonedir)
+            puts "    Cloning repository into #{clonedir}"
+            Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref'])
+          end
+        else
+          puts "    Cloning repository into #{clonedir}"
+          Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref'])
+        end
 
-              # prepare file list to be copied
+        entry['paths'].each do |path|
-              files = FileList[]
+          if File.extname(path['dest']) != ""
-              path['src'].each do |src|
+            if path['src'].size > 1
-                if "#{src}".start_with?("!")
+              raise "Cannot use file destination #{path['dest']} with multiple sources"
-                  files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!")))
+            end
+            FileUtils.mkdir_p File.dirname(path['dest'])
+          else
+            FileUtils.mkdir_p path['dest']
+          end
+
+          puts "    Copying files"
+
+          # prepare file list to be copied
+          files = FileList[]
+          path['src'].each do |src|
+            if "#{src}".start_with?("!")
+              files.exclude(File.join(clonedir, "/"+"#{src}".delete_prefix("!")))
+            else
+              files.include(File.join(clonedir, "/#{src}"))
+            end
+          end
+
+          files.each do |file|
+            FetchRemote.copy(file, path['dest']) do |s, d|
+              s = File.realpath(s)
+              # traverse source directory
+              FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent|
+                file_clean = ent.path.delete_prefix(clonedir).split("/").drop(1).join("/")
+                destent = FileUtils::Entry_.new(d, ent.rel, false)
+                puts "      #{file_clean} => #{destent.path}"
+
+                if File.file?(destent.path)
+                  fmp = FrontMatterParser::Parser.parse_file(destent.path)
+                  if fmp['fetch_remote'].nil?
+                    raise "Local file #{destent.path} already exists"
+                  end
+                  line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil)
+                  lines = File.readlines(ent.path)[line_start..line_end]
+                  File.open(destent.path, "a") { |fow| fow.puts lines.join }
                 else
-                  files.include(File.join(ztmpdir, "*/#{src}"))
+                  ent.copy destent.path
                 end
-              end
 
-              files.each do |file|
+                next unless File.file?(ent.path) && File.extname(ent.path) == ".md"
-                FetchRemote.copy(file, path['dest']) do |s, d|
+                # set edit and issue url and remote info for markdown files in site config defaults
-                  s = File.realpath(s)
+                edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}"
-                  # traverse source directory
+                issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)"
-                  FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent|
+                puts "        edit_url:  #{edit_url}"
-                    file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/")
+                puts "        issue_url: #{issue_url}"
-                    destent = FileUtils::Entry_.new(d, ent.rel, false)
+                site.config['defaults'] << {
-                    puts "      #{file_clean} => #{destent.path}"
+                  "scope" => { "path" => destent.path },
-
+                  "values" => {
-                    if File.file?(destent.path)
+                    "edit_url" => edit_url,
-                      fmp = FrontMatterParser::Parser.parse_file(destent.path)
+                    "issue_url" => issue_url
-                      if fmp['fetch_remote'].nil?
+                  },
-                        raise "Local file #{destent.path} already exists"
+                }
-                      end
+              end, proc do |_| end)
-                      line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil)
-                      lines = File.readlines(ent.path)[line_start..line_end]
-                      File.open(destent.path, "a") { |fow| fow.puts lines.join }
-                    else
-                      ent.copy destent.path
-                    end
-
-                    next unless File.file?(ent.path) && File.extname(ent.path) == ".md"
-                    # set edit and issue url and remote info for markdown files in site config defaults
-                    edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}"
-                    issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)"
-                    puts "        edit_url:  #{edit_url}"
-                    puts "        issue_url: #{issue_url}"
-                    site.config['defaults'] << {
-                      "scope" => { "path" => destent.path },
-                      "values" => {
-                        "edit_url" => edit_url,
-                        "issue_url" => issue_url
-                      },
-                    }
-                  end, proc do |_| end)
-                end
-              end
             end
           end
         end