diff --git a/Dockerfile b/Dockerfile index 0784f49836..2d4f4a9c90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,7 @@ ARG JEKYLL_ENV ARG DOCS_URL ENV TARGET=/out RUN --mount=type=bind,target=.,rw \ + --mount=type=cache,target=/tmp/docker-docs-clone \ --mount=type=cache,target=/src/.jekyll-cache <= 2.0.2, < 5.0) - archive-zip (0.12.0) - io-like (~> 0.3.0) - chef-utils (17.10.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + chef-utils (18.0.161) concurrent-ruby colorator (1.1.0) concurrent-ruby (1.1.10) @@ -18,6 +16,9 @@ GEM ffi (1.15.5) forwardable-extended (2.6.0) front_matter_parser (1.0.1) + git (1.12.0) + addressable (~> 2.8) + rchardet (~> 1.8) html-proofer (3.19.4) addressable (~> 2.3) mercenary (~> 0.3) @@ -29,7 +30,6 @@ GEM http_parser.rb (0.8.0) i18n (1.12.0) concurrent-ruby (~> 1.0) - io-like (0.3.1) jekyll (4.2.2) addressable (~> 2.4) colorator (~> 1.0) @@ -92,13 +92,14 @@ GEM pathutil (0.16.2) forwardable-extended (~> 2.6) posix-spawn (0.3.15) - public_suffix (4.0.7) + public_suffix (5.0.0) racc (1.6.0) rainbow (3.1.1) rake (13.0.6) - rb-fsevent (0.11.1) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) + rchardet (1.8.0) rexml (3.2.5) rouge (3.27.0) safe_yaml (1.0.5) @@ -118,8 +119,8 @@ PLATFORMS x86_64-linux DEPENDENCIES - archive-zip (= 0.12.0) front_matter_parser (= 1.0.1) + git (= 1.12.0) html-proofer (= 3.19.4) jekyll (= 4.2.2) jekyll-last-modified-at diff --git a/_plugins/fetch_remote.rb b/_plugins/fetch_remote.rb index 57ed8f77bb..3a8b69c718 100644 --- a/_plugins/fetch_remote.rb +++ b/_plugins/fetch_remote.rb @@ -1,9 +1,8 @@ -require 'archive/zip' require 'front_matter_parser' +require 'git' require 'jekyll' require 'json' require 'octopress-hooks' -require 'open-uri' require 'rake' require_relative 'util.rb' @@ -12,14 +11,6 @@ module Jekyll class FetchRemote < Octopress::Hooks::Site priority :highest - def self.download(url, dest) - uri = URI.parse(url) - result = File.join(dest, File.basename(uri.path)) - puts " Downloading #{url}" - IO.copy_stream(URI.open(url), result) - return result - end - def self.copy(src, dest) if (tmp = Array.try_convert(src)) tmp.each do |s| @@ -50,75 +41,86 @@ module Jekyll beginning_time = Time.now puts "Starting plugin fetch_remote.rb..." site.config['fetch-remote'].each do |entry| - puts " Repo #{entry['repo']} (#{entry['ref']})" - Dir.mktmpdir do |tmpdir| - tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) - Dir.mktmpdir do |ztmpdir| - puts " Extracting #{tmpfile}" - Archive::Zip.extract( - tmpfile, - ztmpdir, - :create => true - ) - entry['paths'].each do |path| - if File.extname(path['dest']) != "" - if path['src'].size > 1 - raise "Cannot use file destination #{path['dest']} with multiple sources" - end - FileUtils.mkdir_p File.dirname(path['dest']) - else - FileUtils.mkdir_p path['dest'] - end + puts " Repo #{entry['repo']}" - puts " Copying files" + gituri = Git::URL.parse(entry['repo']) + clonedir = "#{Dir.tmpdir}/docker-docs-clone#{gituri.path}" + if Dir.exist?(clonedir) + puts " Opening #{clonedir}" + begin + git = Git.open(clonedir) + git.chdir do + puts " Fetching #{entry['ref']}" + git.checkout(entry['ref']) + git.fetch + end + rescue => e + FileUtils.rm_rf(clonedir) + puts " Cloning repository into #{clonedir}" + Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref']) + end + else + puts " Cloning repository into #{clonedir}" + Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref']) + end - # prepare file list to be copied - files = FileList[] - path['src'].each do |src| - if "#{src}".start_with?("!") - files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) + entry['paths'].each do |path| + if File.extname(path['dest']) != "" + if path['src'].size > 1 + raise "Cannot use file destination #{path['dest']} with multiple sources" + end + FileUtils.mkdir_p File.dirname(path['dest']) + else + FileUtils.mkdir_p path['dest'] + end + + puts " Copying files" + + # prepare file list to be copied + files = FileList[] + path['src'].each do |src| + if "#{src}".start_with?("!") + files.exclude(File.join(clonedir, "/"+"#{src}".delete_prefix("!"))) + else + files.include(File.join(clonedir, "/#{src}")) + end + end + + files.each do |file| + FetchRemote.copy(file, path['dest']) do |s, d| + s = File.realpath(s) + # traverse source directory + FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| + file_clean = ent.path.delete_prefix(clonedir).split("/").drop(1).join("/") + destent = FileUtils::Entry_.new(d, ent.rel, false) + puts " #{file_clean} => #{destent.path}" + + if File.file?(destent.path) + fmp = FrontMatterParser::Parser.parse_file(destent.path) + if fmp['fetch_remote'].nil? + raise "Local file #{destent.path} already exists" + end + line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) + lines = File.readlines(ent.path)[line_start..line_end] + File.open(destent.path, "a") { |fow| fow.puts lines.join } else - files.include(File.join(ztmpdir, "*/#{src}")) + ent.copy destent.path end - end - files.each do |file| - FetchRemote.copy(file, path['dest']) do |s, d| - s = File.realpath(s) - # traverse source directory - FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| - file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/") - destent = FileUtils::Entry_.new(d, ent.rel, false) - puts " #{file_clean} => #{destent.path}" - - if File.file?(destent.path) - fmp = FrontMatterParser::Parser.parse_file(destent.path) - if fmp['fetch_remote'].nil? - raise "Local file #{destent.path} already exists" - end - line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) - lines = File.readlines(ent.path)[line_start..line_end] - File.open(destent.path, "a") { |fow| fow.puts lines.join } - else - ent.copy destent.path - end - - next unless File.file?(ent.path) && File.extname(ent.path) == ".md" - # set edit and issue url and remote info for markdown files in site config defaults - edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" - issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" - puts " edit_url: #{edit_url}" - puts " issue_url: #{issue_url}" - site.config['defaults'] << { - "scope" => { "path" => destent.path }, - "values" => { - "edit_url" => edit_url, - "issue_url" => issue_url - }, - } - end, proc do |_| end) - end - end + next unless File.file?(ent.path) && File.extname(ent.path) == ".md" + # set edit and issue url and remote info for markdown files in site config defaults + edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" + issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" + puts " edit_url: #{edit_url}" + puts " issue_url: #{issue_url}" + site.config['defaults'] << { + "scope" => { "path" => destent.path }, + "values" => { + "edit_url" => edit_url, + "issue_url" => issue_url + }, + } + end, proc do |_| end) end end end