kubernetes
/
website
mirror of https://github.com/kubernetes/website.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
website/update-imported-docs/update-imported-docs.py

256 lines
8.6 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
##
# This script was tested with Python 3.7.4, Go 1.12, and PyYAML 5.1.2
# installed in a virtual environment.
# This script assumes you have the Python package manager 'pip' installed.
#
# This script updates the generated reference documentation.
# See https://kubernetes.io/docs/contribute/generate-ref-docs/kubernetes-components/
# for further details.
#
# This script checks to make sure Go and PyYAML have been installed.
# The reference docs are generated by a Go command so the Go binary must be
# in your PATH.
#
# A temp "work_dir" is created and is the path where repos will be cloned.
# The work_dir is printed out so you can remove it
# when you no longer need the contents.
# This work_dir will temporarily become the GOPATH.
#
# To execute the script from the website/update-imported-docs directory:
# ./update-imported-docs.py <config_file>
# Config files:
# reference.yml use this to update the reference docs
# release.yml use this to auto-generate/import release notes
##
import argparse
import glob
import os
import re
import shutil
import subprocess
import sys
import tempfile
error_msgs = []
# pip should be installed when Python is installed, but just in case...
if not shutil.which('pip'):
error_msgs.append("Install pip so you can install PyYAML. https://pip.pypa.io/en/stable/installing")
reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'])
installed_packages = [r.decode().split('==')[0] for r in reqs.split()]
if 'PyYAML' not in installed_packages:
error_msgs.append(
"Please ensure the PyYAML package is installed; see https://pypi.org/project/PyYAML")
else:
import yaml
if not shutil.which('go'):
error_msgs.append(
"Go must be installed. See https://golang.org/doc/install")
def process_links(content, remote_prefix, sub_path):
"""Process markdown links found in the docs."""
def analyze(match_obj):
ankor = match_obj.group('ankor')
target = match_obj.group('target')
if not (target.startswith("https://") or
target.startswith("mailto:") or
target.startswith("#")):
if target.startswith("/"):
target_list = remote_prefix, target[1:]
target = "/".join(target_list)
else:
target_list = remote_prefix, sub_path, target
target = "/".join(target_list)
return "[%s](%s)" % (ankor, target)
# Links are in the form '[text](url)'
link_regex = re.compile(r"\[(?P<ankor>.*)\]\((?P<target>.*)\)")
content = re.sub(link_regex, analyze, content)
h1_regex = re.compile("^(# .*)?\n")
content = re.sub(h1_regex, "", content)
return content
def process_kubectl_links(content):
"""Update markdown links found in the SeeAlso section of kubectl page.
Example:[kubectl annotate](/docs/reference/generated/kubectl/kubectl-commands#annotate)
"""
def analyze(match_obj):
ankor = match_obj.group('ankor')
target = match_obj.group('target')
if target.endswith(".md") and target.startswith("kubectl"):
ankor_list = ankor.split("kubectl ")
target = "/docs/reference/generated/kubectl/kubectl-commands" + "#" + \
ankor_list[1]
return "[%s](%s)" % (ankor, target)
# Links are in the form '[text](url)'
link_regex = re.compile(r"\[(?P<ankor>.*)\]\((?P<target>.*?)\)")
content = re.sub(link_regex, analyze, content)
return content
def process_file(src, dst, repo_path, repo_dir, root_dir, gen_absolute_links):
"""Process a file element.
:param src: A string containing the relative path of a source file. The
string may contain wildcard characters such as '*' or '?'.
:param dst: The path for the destination file. The string can be a
directory name or a file name.
:param repo_path:
:param repo_dir:
:param root_dir:
:param gen_absolute_links:
"""
pattern = os.path.join(repo_dir, repo_path, src)
dst_path = os.path.join(root_dir, dst)
for src in glob.glob(pattern):
# we don't dive into subdirectories
if not os.path.isfile(src):
print("[Error] skipping non-regular path {}".format(src))
continue
content = ""
try:
with open(src, "r") as srcFile:
content = srcFile.read()
except Exception as ex:
print("[Error] failed in reading source file: ".format(ex))
continue
dst = dst_path
if dst_path.endswith("/"):
base_name = os.path.basename(src)
dst = os.path.join(dst, base_name)
try:
print("Writing doc: " + dst)
with open(dst, "w") as dstFile:
if gen_absolute_links:
src_dir = os.path.dirname(src)
remote_prefix = repo_path + "/tree/master"
content = process_links(content, remote_prefix, src_dir)
if dst.endswith("kubectl.md"):
print("Processing kubectl links")
content = process_kubectl_links(content)
dstFile.write(content)
except Exception as ex:
print("[Error] failed in writing target file {}: {}".format(dst, ex))
continue
def parse_input_args():
"""
Parse command line argument
'config_file' is only one argument; it should be one of the YAML
files in this same directory
:return: parsed argument
"""
parser = argparse.ArgumentParser()
parser.add_argument('config_file', type=str,
help="reference.yml to generate reference docs; "
"release.yml to generate release notes")
return parser.parse_args()
def main():
"""The main entry of the program."""
if len(error_msgs) > 0:
for msg in error_msgs:
print(msg + "\n")
return -2
# first parse input argument
in_args = parse_input_args()
config_file = in_args.config_file
print("config_file is {}".format(config_file))
curr_dir = os.path.dirname(__file__)
print("curr_dir {}".format(curr_dir))
root_dir = os.path.realpath(os.path.join(curr_dir, '..'))
print("root_dir {}".format(root_dir))
try:
config_data = yaml.full_load(open(config_file, 'r'))
except Exception as ex:
# to catch when a user specifies a file that does not exist
print("[Error] failed in loading config file - {}".format(str(ex)))
return -2
os.chdir(root_dir)
# create the temp work_dir
try:
print("Making temp work_dir")
work_dir = tempfile.mkdtemp()
except OSError as ose:
print("[Error] Unable to create temp work_dir {}; error: {}"
.format(work_dir, ose))
return -2
for repo in config_data["repos"]:
if "name" not in repo:
print("[Error] repo missing name")
continue
repo_name = repo["name"]
if "remote" not in repo:
print("[Error] repo {} missing repo path".format(repo_name))
continue
repo_remote = repo["remote"]
remote_regex = re.compile(r"^https://(?P<prefix>.*)\.git$")
matches = remote_regex.search(repo_remote)
if not matches:
print("[Error] repo path for {} is invalid".format(repo_name))
continue
repo_path = os.path.join("src", matches.group('prefix'))
os.chdir(work_dir)
print("Cloning repo {}".format(repo_name))
cmd = "git clone --depth=1 -b {0} {1} {2}".format(
repo["branch"], repo_remote, repo_path)
res = subprocess.call(cmd, shell=True)
if res != 0:
print("[Error] failed in cloning repo {}".format(repo_name))
continue
os.chdir(repo_path)
if "generate-command" in repo:
gen_cmd = repo["generate-command"]
gen_cmd = "export GOPATH=" + work_dir + "\n" + gen_cmd
print("Generating docs for {} with {}".format(repo_name, gen_cmd))
res = subprocess.call(gen_cmd, shell=True)
if res != 0:
print("[Error] failed in generating docs for {}".format(
repo_name))
continue
os.chdir(root_dir)
for f in repo["files"]:
process_file(f['src'], f['dst'], repo_path, work_dir, root_dir,
"gen-absolute-links" in repo)
print("Completed docs update. Now run the following command to commit:\n\n"
" git add .\n"
" git commit -m <comment>\n"
" git push\n"
" delete temp dir {} when done ".format(work_dir))
if __name__ == '__main__':
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink