istio
/
istio.io
mirror of https://github.com/istio/istio.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update common files (#4870) 

- Enable a bunch of linting rules in the repo, and addressed all the linting
errors
This commit is contained in:
Martin Taillefer 2019-08-29 08:09:32 -07:00 committed by Istio Automation
parent 118eb9c1fe
commit a48f05a3e9
16 changed files with 566 additions and 739 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -66,6 +66,7 @@ RUN = docker run -t -i --sig-proxy=true -u $(UID) --rm \
-e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \
-v /etc/passwd:/etc/passwd:ro \
-v $(readlink /etc/localtime):/etc/localtime:ro \
-v /var/run/docker.sock:/var/run/docker.sock \
$(CONTAINER_OPTIONS) \
--mount type=bind,source="$(PWD)",destination="/work" \
--mount type=volume,source=istio-go-mod,destination="/go/pkg/mod" \

4
Makefile.core.mk
View File

@ -23,7 +23,7 @@ clean_public:
clean: clean_public
@rm -fr resources .htmlproofer tmp
lint: clean_public gen_nominify
lint: clean_public gen_nominify lint-copyright-banner lint-python lint-yaml lint-dockerfiles
@scripts/lint_site.sh
serve: build
@ -55,8 +55,6 @@ netlify_archive:
archive:
@scripts/gen_archive_site.sh "$(baseurl)"
prow: lint
update_ref_docs:
@scripts/grab_reference_docs.sh

2
common/.commonfiles.sha
View File

@ -1 +1 @@
924cf1256448b0869bf55646525e1439ec11f85c
02f70e0b143bfe50002d3915e10f35571463dd65

39
scripts/build_site.sh
View File

@ -1,4 +1,19 @@
#!/usr/bin/env sh
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
mkdir -p generated/css generated/js generated/img tmp/js
@ -7,6 +22,26 @@ sass src/sass/_all.scss all.css -s compressed --no-source-map
mv all.css* generated/css
tsc
babel --source-maps --minified --no-comments --presets minify tmp/js/constants.js tmp/js/utils.js tmp/js/kbdnav.js tmp/js/themes.js tmp/js/menu.js tmp/js/header.js tmp/js/sidebar.js tmp/js/tabset.js tmp/js/prism.js tmp/js/codeBlocks.js tmp/js/links.js tmp/js/scroll.js tmp/js/overlays.js tmp/js/lang.js tmp/js/callToAction.js --out-file generated/js/all.min.js
babel --source-maps --minified --no-comments --presets minify tmp/js/themes_init.js --out-file generated/js/themes_init.min.js
babel --source-maps --minified --no-comments --presets minify \
tmp/js/constants.js \
tmp/js/utils.js \
tmp/js/kbdnav.js \
tmp/js/themes.js \
tmp/js/menu.js \
tmp/js/header.js \
tmp/js/sidebar.js \
tmp/js/tabset.js \
tmp/js/prism.js \
tmp/js/codeBlocks.js \
tmp/js/links.js \
tmp/js/scroll.js \
tmp/js/overlays.js \
tmp/js/lang.js \
tmp/js/callToAction.js \
--out-file generated/js/all.min.js
babel --source-maps --minified --no-comments --presets minify \
tmp/js/themes_init.js \
--out-file generated/js/themes_init.min.js
svgstore -o generated/img/icons.svg src/icons/**/*.svg

52
scripts/check_dockerfiles.sh
View File

@ -1,52 +0,0 @@
#!/bin/bash
# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
#
# The original version of this file is located in the https://github.com/istio/common-files repo.
# If you're looking at this file in a different repo and want to make a change, please go to the
# common-files repo, make the change there and check it in. Then come back to this repo and run
# "make updatecommon".
# Copyright 2019 Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}")" && pwd )"
ROOTDIR=$(dirname "${SCRIPTPATH}")
cd "${ROOTDIR}"
CD_TMPFILE=$(mktemp /tmp/check_dockerfile.XXXXXX)
HL_TMPFILE=$(mktemp /tmp/hadolint.XXXXXX)
# shellcheck disable=SC2044
for df in $(find "${ROOTDIR}" -path "${ROOTDIR}/vendor" -prune -o -name 'Dockerfile*'); do
docker run --rm -i hadolint/hadolint:v1.17.1 < "$df" > "${HL_TMPFILE}"
if [ "" != "$(cat "${HL_TMPFILE}")" ]
then
{
echo "$df:"
cut -d":" -f2- < "${HL_TMPFILE}"
echo
} >> "${CD_TMPFILE}"
fi
done
rm -f "${HL_TMPFILE}"
if [ "" != "$(cat "${CD_TMPFILE}")" ]; then
cat "${CD_TMPFILE}"
rm -f "${CD_TMPFILE}"
exit 1
fi
rm -f "${CD_TMPFILE}"

83
scripts/check_license.sh
View File

@ -1,83 +0,0 @@
#!/bin/bash
# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
#
# The original version of this file is located in the https://github.com/istio/common-files repo.
# If you're looking at this file in a different repo and want to make a change, please go to the
# common-files repo, make the change there and check it in. Then come back to this repo and run
# "make updatecommon".
# Copyright 2018 Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOTDIR=$(dirname "${SCRIPTPATH}")
cd "${ROOTDIR}"
ADD_LICENSE=$1
THISYEAR=$(date +"%Y")
if [[ $ADD_LICENSE == true ]]; then
echo "Check License script is running in ADD_LICENSE mode. It will automatically add any missing licenses for you."
fi
ret=0
for fn in $(find "${ROOTDIR}" -type f \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' \) | grep -v vendor | grep -v testdata); do
if [[ $fn == *.pb.go ]];then
continue
fi
if head -20 "$fn" | grep "auto\\-generated" > /dev/null; then
continue
fi
if head -20 "$fn" | grep "DO NOT EDIT" > /dev/null; then
continue
fi
if head -20 "$fn" | grep "Code generated by go-bindata" > /dev/null; then
continue
fi
if ! head -20 "$fn" | grep "Apache License, Version 2" > /dev/null; then
if [[ $ADD_LICENSE == true ]]; then
echo "// Copyright ${THISYEAR} Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the \"License\");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an \"AS IS\" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
$(cat "${fn}")" > "${fn}"
else
echo "${fn} missing license"
ret=$((ret+1))
fi
fi
if ! head -20 "$fn" | grep Copyright > /dev/null; then
echo "${fn} missing Copyright"
ret=$((ret+1))
fi
done
exit $ret

65
scripts/gen_archive_site.sh
View File

@ -1,7 +1,22 @@
#! /bin/bash
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Build the archive site
#
set -e
BASEURL="$1"
@ -22,51 +37,45 @@ TOBUILD_JEKYLL=(
v0.1:release-0.1
)
# Prereqs
npm install sass sass-lint typescript tslint @babel/cli @babel/core svgstore-cli html-minifier --save-dev
npm install babel-preset-minify --save-dev
# Prepare
TMP=$(mktemp -d)
mkdir ${TMP}/archive
mkdir "${TMP}/archive"
GITDIR=istio.io
rm -fr ${GITDIR}
git clone https://github.com/istio/istio.io.git
cd ${GITDIR}
for rel in "${TOBUILD[@]}"
do
NAME=$(echo $rel | cut -d : -f 1)
TAG=$(echo $rel | cut -d : -f 2)
URL=$(echo ${BASEURL}/${NAME})
for rel in "${TOBUILD[@]}"; do
NAME=$(echo "$rel" | cut -d : -f 1)
TAG=$(echo "$rel" | cut -d : -f 2)
URL=${BASEURL}/${NAME}
echo "### Building '${NAME}' from ${TAG} for ${URL}"
git checkout ${TAG}
git checkout "${TAG}"
scripts/gen_site.sh ${URL}
scripts/gen_site.sh "${URL}"
mv public ${TMP}/archive/${NAME}
echo "- name: \"${NAME}\"" >> ${TMP}/archives.yml
mv public "${TMP}/archive/${NAME}"
echo "- name: \"${NAME}\"" >> "${TMP}/archives.yml"
git clean -f
done
for rel in "${TOBUILD_JEKYLL[@]}"
do
NAME=$(echo $rel | cut -d : -f 1)
TAG=$(echo $rel | cut -d : -f 2)
URL=$(echo ${BASEURL}/${NAME})
for rel in "${TOBUILD_JEKYLL[@]}"; do
NAME=$(echo "$rel" | cut -d : -f 1)
TAG=$(echo "$rel" | cut -d : -f 2)
URL=${BASEURL}/${NAME}
echo "### Building '${NAME}' from ${TAG} for ${URL}"
git checkout ${TAG}
git checkout "${TAG}"
echo "baseurl: ${URL}" > config_override.yml
bundle install
bundle exec jekyll build --config _config.yml,config_override.yml
mv _site ${TMP}/archive/${NAME}
echo "- name: \"${NAME}\"" >> ${TMP}/archives.yml
mv _site "${TMP}/archive/${NAME}"
echo "- name: \"${NAME}\"" >> "${TMP}/archives.yml"
git clean -f
done
@ -81,14 +90,14 @@ sed -i 's/preliminary: true/preliminary: false/g' data/args.yml
sed -i 's/archive_landing: false/archive_landing: true/g' data/args.yml
# Grab the state
cp ${TMP}/archives.yml data
cp "${TMP}/archives.yml" data
scripts/build_site.sh
scripts/gen_site.sh "$1"
mv public/* ${TMP}/archive
rm -fr ${GITDIR} public
mv ${TMP}/archive public
rm -fr ${TMP}
mv public/* "${TMP}/archive"
rm -fr "${GITDIR}" public
mv "${TMP}/archive" public
rm -fr "${TMP}"
echo "All done!"

18
scripts/gen_site.sh
View File

@ -1,10 +1,24 @@
#! /bin/bash
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
hugo version
if [[ "$2" == "-no_minify" ]]
then
if [[ "$2" == "-no_minify" ]]; then
hugo --baseURL "$1"
else
hugo --minify --baseURL "$1"

46
scripts/grab_reference_docs.sh
View File

@ -1,13 +1,27 @@
#!/bin/bash
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script copies generated .pb.html files, which contain reference docs for protos, and installs
# them in their targeted location within the content/docs/reference tree of this repo. Each .pb.html file contains a
# them in their targeted location within the content/en/docs/reference tree of this repo. Each .pb.html file contains a
# line that indicates the target directory location. The line is of the form:
#
# location: https://istio.io/docs/reference/...
#
# Additionally, this script also builds Istio components and runs them to extract their command-line docs which it
# copies to content/docs/reference/commands.
# copies to content/en/docs/reference/commands.
# The repos to mine for docs, just add new entries here to pull in more repos.
REPOS=(
@ -46,14 +60,6 @@ echo "WORK_DIR =" "${WORK_DIR}"
#####################
pushd () {
command pushd "$@" > /dev/null
}
popd () {
command popd > /dev/null
}
# Given the name of a .pb.html file, extracts the $location marker and then proceeds to
# copy the file to the corresponding content/en/docs/ hierarchy.
locate_file() {
@ -82,8 +88,8 @@ locate_file() {
REPOX=${REPO_URL/.git/}
REPOX=${REPOX//\//\\\/}
sed -i -e 's/title: /WARNING: THIS IS AN AUTO-GENERATED FILE, DO NOT EDIT. PLEASE MODIFY THE ORIGINAL SOURCE IN THE '${REPOX}' REPO\'$'\n''title: /g' "${ROOTDIR}/content/en/docs${PP}/${FN}/index.html"
sed -i -e 's/title: /source_repo: '${REPOX}'\'$'\n''title: /g' "${ROOTDIR}/content/en/docs${PP}/${FN}/index.html"
sed -i -e 's/title: /WARNING: THIS IS AN AUTO-GENERATED FILE, DO NOT EDIT. PLEASE MODIFY THE ORIGINAL SOURCE IN THE '"${REPOX}"' REPO\'$'\n''title: /g' "${ROOTDIR}/content/en/docs${PP}/${FN}/index.html"
sed -i -e 's/title: /source_repo: '"${REPOX}"'\'$'\n''title: /g' "${ROOTDIR}/content/en/docs${PP}/${FN}/index.html"
}
handle_doc_scraping() {
@ -94,13 +100,13 @@ handle_doc_scraping() {
echo " INPUT REPO: ${REPO_URL}@${REPO_BRANCH}"
git clone --depth=1 -q -b ${REPO_BRANCH} ${REPO_URL} ${DEST_DIR}
git clone --depth=1 -q -b "${REPO_BRANCH}" "${REPO_URL}" "${DEST_DIR}"
# delete the vendor directory so we don't get .pb.html out of there
rm -fr "${DEST_DIR}/vendor"
for f in `find "${DEST_DIR}" -type f -name '*.pb.html'`; do
locate_file ${f}
for f in $(find "${DEST_DIR}" -type f -name '*.pb.html'); do
locate_file "${f}"
done
rm -fr "${DEST_DIR}"
@ -119,8 +125,8 @@ handle_components() {
git clone --depth=1 -q -b "${REPO_BRANCH}" "${REPO_URL}"
pushd "${REPO_NAME}" || exit
pushd "${COMP_PATH}" || exit
pushd "${REPO_NAME}" >dev/null || exit
pushd "${COMP_PATH}" >dev/null || exit
go build -o "${COMP_NAME}"
mkdir -p "${COMP_OUTPUT_DIR}/${COMP_NAME}"
@ -131,8 +137,8 @@ handle_components() {
sed -i -e 's/title: /WARNING: THIS IS AN AUTO-GENERATED FILE, DO NOT EDIT. PLEASE MODIFY THE ORIGINAL SOURCE IN THE https:\/\/github.com\/istio\/istio REPO\'$'\n''title: /g' "${COMP_OUTPUT_DIR}/${COMP_NAME}/index.html"
sed -i -e 's/title: /source_repo: https:\/\/github.com\/istio\/istio\'$'\n''title: /g' "${COMP_OUTPUT_DIR}/${COMP_NAME}/index.html"
popd || exit
popd || exit
popd >dev/null || exit
popd >dev/null || exit
rm -fr "${REPO_NAME}"
done
@ -143,7 +149,7 @@ find "${ROOTDIR}/content/en/docs/reference" -name '*.html' -type f -print0 | xar
# Prepare the work directory
mkdir -p "${WORK_DIR}"
pushd "${WORK_DIR}" || exit
pushd "${WORK_DIR}" >dev/null || exit
#echo "Handling doc scraping"
handle_doc_scraping

94
scripts/lint_site.sh
View File

@ -1,5 +1,21 @@
#!/bin/bash
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
FAILED=0
echo -ne "mdspell "
@ -22,123 +38,99 @@ check_content() {
TMP=$(mktemp -d)
# make the tmp dir
mkdir -p ${TMP}
mkdir -p "${TMP}"
# create a throwaway copy of the content
cp -R ${DIR} ${TMP}
cp .spelling ${TMP}
cp mdl_style.rb ${TMP}
cp -R "${DIR}" "${TMP}"
cp .spelling "${TMP}"
cp mdl_style.rb "${TMP}"
# replace the {{< text >}} shortcodes with ```plain
find ${TMP} -type f -name \*.md -exec sed -E -i "s/\\{\\{< text .*>\}\}/\`\`\`plain/g" {} ";"
find "${TMP}" -type f -name \*.md -exec sed -E -i "s/\\{\\{< text .*>\}\}/\`\`\`plain/g" {} ";"
# replace the {{< /text >}} shortcodes with ```
find ${TMP} -type f -name \*.md -exec sed -E -i "s/\\{\\{< \/text .*>\}\}/\`\`\`/g" {} ";"
find "${TMP}" -type f -name \*.md -exec sed -E -i "s/\\{\\{< \/text .*>\}\}/\`\`\`/g" {} ";"
# elide url="*"
find ${TMP} -type f -name \*.md -exec sed -E -i "s/url=\".*\"/URL/g" {} ";"
find "${TMP}" -type f -name \*.md -exec sed -E -i "s/url=\".*\"/URL/g" {} ";"
# elide link="*"
find ${TMP} -type f -name \*.md -exec sed -E -i "s/link=\".*\"/LINK/g" {} ";"
find "${TMP}" -type f -name \*.md -exec sed -E -i "s/link=\".*\"/LINK/g" {} ";"
# switch to the temp dir
pushd ${TMP} >/dev/null
pushd "${TMP}" >/dev/null
mdspell ${LANG} --ignore-acronyms --ignore-numbers --no-suggestions --report `find content/en -type f -name '*.md'`
if [[ "$?" != "0" ]]
then
if ! mdspell "${LANG}" --ignore-acronyms --ignore-numbers --no-suggestions --report $(find content/en -type f -name '*.md'); then
echo "To learn how to address spelling errors, please see https://istio.io/about/contribute/creating-and-editing-pages/#linting"
FAILED=1
fi
mdl --ignore-front-matter --style mdl_style.rb .
if [[ "$?" != "0" ]]
then
if ! mdl --ignore-front-matter --style mdl_style.rb .; then
FAILED=1
fi
grep -nr -e "(https://istio.io" .
if [[ "$?" == "0" ]]
then
if grep -nr -e "(https://istio.io" .; then
echo "Ensure markdown content uses relative references to istio.io"
FAILED=1
fi
grep -nr -e "(https://preliminary.istio.io" .
if [[ "$?" == "0" ]]
then
if grep -nr -e "(https://preliminary.istio.io" .; then
echo "Ensure markdown content doesn't contain references to preliminary.istio.io"
FAILED=1
fi
grep -nr -e "https://github.com/istio/istio/blob/" .
if [[ "$?" == "0" ]]
then
if grep -nr -e https://github.com/istio/istio/blob/ .; then
echo "Ensure markdown content uses {{< github_blob >}}"
FAILED=1
fi
grep -nr -e "https://github.com/istio/istio/tree/" .
if [[ "$?" == "0" ]]
then
if grep -nr -e https://github.com/istio/istio/tree/ .; then
echo "Ensure markdown content uses {{< github_tree >}}"
FAILED=1
fi
grep -nr -e "https://raw.githubusercontent.com/istio/istio/" .
if [[ "$?" == "0" ]]
then
if grep -nr -e https://raw.githubusercontent.com/istio/istio/ .; then
echo "Ensure markdown content uses {{< github_file >}}"
FAILED=1
fi
# go back whence we came
popd >/dev/null
popd >/dev/null
# cleanup
rm -fr ${TMP}
rm -fr "${TMP}"
}
check_content content --en-us
for f in `find ./content/en -type f \( -name '*.html' -o -name '*.md' \)`
do
grep -H -n -e "“" ${f}
if [[ "$?" == "0" ]]
then
for f in $(find ./content/en -type f \( -name '*.html' -o -name '*.md' \)); do
# shellcheck disable=SC1111
if grep -H -n -e '“' "${f}"; then
echo "Ensure content only uses standard quotation marks and not “"
FAILED=1
fi
done
for f in `find ./public -type f -name '*.html'`
do
grep -H -n -i -e "blockquote" ${f}
if [[ "$?" == "0" ]]
then
for f in $(find ./public -type f -name '*.html'); do
if grep -H -n -i -e blockquote "${f}"; then
echo "Ensure content only uses {{< tip >}}, {{< warning >}}, {{< idea >}}, and {{< quote >}} instead of block quotes"
FAILED=1
fi
grep -H -n -e "\"https://github.*#L[0-9]*\"" ${f}
if [[ "$?" == "0" ]]
then
if grep -H -n -e "\"https://github.*#L[0-9]*\"" "${f}"; then
echo "Ensure content doesn't use links to specific lines in GitHub files as those are too brittle"
FAILED=1
fi
done
sass-lint -c sass-lint.yml --verbose 'src/sass/**/*.scss'
sass-lint -c sass-lint.yml --verbose src/sass/**/*.scss
tslint src/ts/*.ts
htmlproofer ./public --assume-extension --check-html --check-external-hash --check-opengraph --timeframe 2d --storage-dir .htmlproofer --url-ignore "/localhost/,/github.com/istio/istio.io/edit/,/github.com/istio/istio/issues/new/choose/,/groups.google.com/forum/,/www.trulia.com/,/apporbit.com/"
if [[ "$?" != "0" ]]
then
if ! htmlproofer ./public --assume-extension --check-html --check-external-hash --check-opengraph --timeframe 2d --storage-dir .htmlproofer --url-ignore "/localhost/,/github.com/istio/istio.io/edit/,/github.com/istio/istio/issues/new/choose/,/groups.google.com/forum/,/www.trulia.com/,/apporbit.com/"; then
FAILED=1
fi
if [[ ${FAILED} -eq 1 ]]
then
if [[ ${FAILED} -eq 1 ]]; then
echo "LINTING FAILED"
exit 1
fi

23
scripts/opt_site.sh
View File

@ -1,8 +1,25 @@
#!/usr/bin/env sh
# Copyright Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Optimize site assets
set -e
svgo --version
npx svgo -r -f content/en
npx svgo -r -f content/zh
npx svgo -r -f src/icons
svgo -r -f content/en
svgo -r -f content/zh
svgo -r -f src/icons

31
scripts/run_gobindata.sh
View File

@ -1,31 +0,0 @@
#!/bin/bash
# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
#
# The original version of this file is located in the https://github.com/istio/common-files repo.
# If you're looking at this file in a different repo and want to make a change, please go to the
# common-files repo, make the change there and check it in. Then come back to this repo and run
# "make updatecommon".
# Copyright 2018 Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOTDIR=$(dirname "${SCRIPTPATH}")
img=gcr.io/istio-testing/build-tools:2019-08-18T16-45-52
docker run -i --sig-proxy=true --rm --entrypoint go-bindata --user "$(id -u)" -v /etc/passwd:/etc/passwd:ro -v "${ROOTDIR}:${ROOTDIR}" -w "$(pwd)" ${img} "$@"

56
scripts/run_gofmt.sh
View File

@ -1,56 +0,0 @@
#!/bin/bash
# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
#
# The original version of this file is located in the https://github.com/istio/common-files repo.
# If you're looking at this file in a different repo and want to make a change, please go to the
# common-files repo, make the change there and check it in. Then come back to this repo and run
# "make updatecommon".
# Copyright 2018 Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Applies requisite code formatters to the source tree
set -e
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOTDIR=$(dirname "${SCRIPTPATH}")
cd "${ROOTDIR}"
# Go format tool to use
# While 'goimports' is preferred we temporarily use 'gofmt' until https://github.com/golang/go/issues/28200 is resolved
GO_FMT_TOOL=goimportsdocker
PKGS=${PKGS:-"."}
if [[ -z ${GO_FILES} ]];then
GO_FILES=$(find "${PKGS}" -type f -name '*.go' ! -name '*.gen.go' ! -name '*.pb.go' ! -name '*mock*.go' | grep -v ./vendor)
fi
# need to pin goimports to align with golangci-lint. SHA is from x/tools repo
if [ $GO_FMT_TOOL = "goimportsdocker" ]; then
GO_IMPORTS_DOCKER="gcr.io/istio-testing/goimports:379209517ffe"
tool="docker run -i --rm -v ${ROOTDIR}:${ROOTDIR} -w ${ROOTDIR} ${GO_IMPORTS_DOCKER} /goimports"
fmt_args="-w -local istio.io"
fi
if [ $GO_FMT_TOOL = "gofmt" ]; then
tool=gofmt
fmt_args="-w"
fi
echo "Formatting the source files"
# shellcheck disable=SC2086
$tool ${fmt_args} ${GO_FILES}
exit $?

40
scripts/run_golangci.sh
View File

@ -1,40 +0,0 @@
#!/bin/bash
# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
#
# The original version of this file is located in the https://github.com/istio/common-files repo.
# If you're looking at this file in a different repo and want to make a change, please go to the
# common-files repo, make the change there and check it in. Then come back to this repo and run
# "make updatecommon".
# Copyright 2019 Istio Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOTDIR=$(dirname "${SCRIPTPATH}")
cd "${ROOTDIR}"
if [[ "$1" == "--fix" ]]
then
FIX="--fix"
fi
# if you want to update this version, also change the version number in .golangci.yml
GOLANGCI_VERSION="v1.16.0"
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b "$GOPATH"/bin "$GOLANGCI_VERSION"
golangci-lint --version
# For tuning and when switching versions PLEASE REFERENCE: https://github.com/istio/istio/issues/14888
env GOGC=25 golangci-lint run ${FIX} -j 8 -v ./...

348
scripts/tablegen.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python
# Copyright 2017,2018 Istio Authors. All Rights Reserved.
# Copyright Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -42,6 +42,7 @@ VALUES_YAML = "values.yaml"
ISTIO_IO_DIR = os.path.abspath(__file__ + "/../../")
CONFIG_INDEX_DIR = "content/en/docs/reference/config/installation-options/index.md"
def endOfTheList(context, lineNum, lastLineNum, totalNum):
flag = 0
valueList = []
@ -55,17 +56,17 @@ def endOfTheList(context, lineNum, lastLineNum, totalNum):
for nextLineNum in range(lineNum + 1, totalNum):
nextLine = context[nextLineNum]
if len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' in nextLine:
if len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' in nextLine:
if whitespaces >= (len(nextLine) - len(nextLine.lstrip())) / 2:
if flag == 0:
valueList.append(currentLine.split(':', 1)[1].strip())
return True, valueList
else:
return True, valueList
elif len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' not in nextLine and len(nextLine.strip()) != 0:
elif len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' not in nextLine and len(nextLine.strip()) != 0:
value = nextLine.replace(' ', '')
valueList.append(value.lstrip('-').strip())
flag += 1;
flag += 1
nextLineNum += 1
if lastLineNum == totalNum - 1 and len(currentLine.lstrip()) != 0 and '#' != currentLine.lstrip()[0]:
@ -73,12 +74,14 @@ def endOfTheList(context, lineNum, lastLineNum, totalNum):
return True, valueList
prdict = collections.defaultdict(list)
def decode_helm_yaml(s):
ret_val = ''
#
# Iterate through all the directories under /istio/install/kubernetes/helm/subcharts
# Iterate through all the directories under /istio/install/kubernetes/helm/subcharts
# and process the configuration options from the respective values.yaml. The
# configuration option name is the name of the directory that contains values.yaml.
# This name will be passed in to the the function process_helm_yaml
@ -100,166 +103,168 @@ def decode_helm_yaml(s):
return ret_val
def process_helm_yaml(values_yaml, option):
ret_val = ''
storekey = ''
desc = ''
newkey = ''
whitespaces = 0
flag = 0
lineNum = 0
newConfigList = []
loaded = None
ret_val = ''
storekey = ''
desc = ''
newkey = ''
whitespaces = 0
flag = 0
lineNum = 0
newConfigList = []
loaded = None
context = linecache.getlines(values_yaml)
totalNum = len(context)
lastLineNum = 0
key = option
count = 0
with open(values_yaml, 'r') as f_v:
d_v = f_v.read()
loaded = yaml.round_trip_load(d_v)
context = linecache.getlines(values_yaml)
totalNum = len(context)
lastLineNum = 0
key = option
for lineNum in range(0, totalNum):
if context[lineNum].strip().startswith('- '):
pass
elif '#' in context[lineNum] and '#' == context[lineNum].lstrip()[0]:
if "Description: " in context[lineNum]:
desc = context[lineNum].strip()
elif ':' in context[lineNum] and '#' != context[lineNum].lstrip()[0]:
lastLineNum = lineNum
if flag == 1:
whitespaces = (len(context[lineNum]) - len(context[lineNum].lstrip())) / 2
periods = key.count('.')
if (option == ''):
while (whitespaces <= periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
else:
while (whitespaces < periods) :
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
flag = 0
count = 0
with open(values_yaml, 'r') as f_v:
d_v = f_v.read()
loaded = yaml.round_trip_load(d_v)
key = key + '.' + context[lineNum].split(':', 1)[0].strip()
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
if isEnd == True:
flag = 1;
for lineNum in range(0, totalNum):
if context[lineNum].strip().startswith('- '):
pass
elif '#' in context[lineNum] and '#' == context[lineNum].lstrip()[0]:
if "Description: " in context[lineNum]:
desc = context[lineNum].strip()
elif ':' in context[lineNum] and '#' != context[lineNum].lstrip()[0]:
lastLineNum = lineNum
if flag == 1:
whitespaces = (len(context[lineNum]) - len(context[lineNum].lstrip())) / 2
periods = key.count('.')
if (option == ''):
while (whitespaces <= periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
else:
while (whitespaces < periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
flag = 0
storekey = key
sk = storekey.split('.', 2)
if len(sk) > 1:
storekey = '.'.join(sk[:1]).lstrip('.')
else:
storekey = '.'.join(sk[:0]).lstrip('.')
key = key + '.' + context[lineNum].split(':', 1)[0].strip()
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
if isEnd:
flag = 1
#
# If we are processing the configurations options within the values.yaml under istio,
# if the options have already been processed (from the subcharts directory), then we
# do not want to process it again. If the configuration option has not been processed
# before, then it is a new configuration option which needs to be processed (for e.g,
# global, istiocoredns)
#
# option == '' - This condition means that we are looking at the values.yaml under the
# istio directory. Hence, the configuration option names will be inside
# the values.yaml file. (On the other hand, for the values.yaml file under
# the subcharts directory, we get the name of the configuration option
# from the name of the directories under the subcharts directory.)
# newConfigList - This list is used to track configuration options in istio/values.yaml
# that haven't been processed before (or that does not have a corresponding
# directory under subcharts directory with values.yaml. E.g: global,
# istiocoredns)
#
# This first condition checks that if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has not already been processed (this
# conditions: "prdict.get(storekey) != None and (storekey in newConfigList)" together
# makes sure that the condition where some parameters for a new configuration option like
# 'global' has been processed and entered into the dictionary 'prdict' is still processed
# because it is in the newConfigList. If a configuration option was processed from
# the values.yaml under the subcharts directory, it will not be in the newConfigList.
# subcharts directory), then go ahead and process the parameters for this option.
#
if option == '' and prdict.get(storekey) != None and (storekey in newConfigList):
pass
#
# This second condition checks if this is the values.yaml file under istio directory, and
# the configuration option to process (storekey) has not been processed (this could
# happen the first time we read a configuration option from the istio/values.yaml file),
# then add this configuration option to the newConfigList to mark it as an option that
# needs to be processed.
#
elif option == '' and prdict.get(storekey) == None:
newConfigList.append(storekey)
#
# This third condition checks if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has already been processed and if
# this is not a new configuration option, (this could happen if we have already
# processed the corresponding values.yaml under the subcharts directory), then ignore
# this configuration option and do not process the values in this file.
#
elif option == '' and prdict.get(storekey) != None:
continue
storekey = key
sk = storekey.split('.', 2)
if len(sk) > 1:
storekey = '.'.join(sk[:1]).lstrip('.')
else:
storekey = '.'.join(sk[:0]).lstrip('.')
if len(context[lastLineNum].lstrip()) != 0 and '#' != context[lastLineNum].lstrip()[0]:
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
#
# If we are processing the configurations options within the values.yaml under istio,
# if the options have already been processed (from the subcharts directory), then we
# do not want to process it again. If the configuration option has not been processed
# before, then it is a new configuration option which needs to be processed (for e.g,
# global, istiocoredns)
#
# option == '' - This condition means that we are looking at the values.yaml under the
# istio directory. Hence, the configuration option names will be inside
# the values.yaml file. (On the other hand, for the values.yaml file under
# the subcharts directory, we get the name of the configuration option
# from the name of the directories under the subcharts directory.)
# newConfigList - This list is used to track configuration options in istio/values.yaml
# that haven't been processed before (or that does not have a corresponding
# directory under subcharts directory with values.yaml. E.g: global,
# istiocoredns)
#
# This first condition checks that if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has not already been processed (this
# conditions: "prdict.get(storekey) != None and (storekey in newConfigList)" together
# makes sure that the condition where some parameters for a new configuration option like
# 'global' has been processed and entered into the dictionary 'prdict' is still processed
# because it is in the newConfigList. If a configuration option was processed from
# the values.yaml under the subcharts directory, it will not be in the newConfigList.
# subcharts directory), then go ahead and process the parameters for this option.
#
if option == '' and prdict.get(storekey) is not None and (storekey in newConfigList):
pass
#
# This second condition checks if this is the values.yaml file under istio directory, and
# the configuration option to process (storekey) has not been processed (this could
# happen the first time we read a configuration option from the istio/values.yaml file),
# then add this configuration option to the newConfigList to mark it as an option that
# needs to be processed.
#
elif option == '' and prdict.get(storekey) is None:
newConfigList.append(storekey)
#
# This third condition checks if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has already been processed and if
# this is not a new configuration option, (this could happen if we have already
# processed the corresponding values.yaml under the subcharts directory), then ignore
# this configuration option and do not process the values in this file.
#
elif option == '' and prdict.get(storekey) is not None:
continue
if (isEnd == True):
flag = 1
keysplit = key.split('.')
for kv in keysplit:
if kv != '':
newkey = newkey + '.' + kv
if len(context[lastLineNum].lstrip()) != 0 and '#' != context[lastLineNum].lstrip()[0]:
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
newkey = newkey.lstrip('.')
if (isEnd):
flag = 1
keysplit = key.split('.')
for kv in keysplit:
if kv != '':
newkey = newkey + '.' + kv
# Filling Description Fields
if ( "." in newkey):
plist = newkey.split('.')
da = None
for item in plist:
desc = ''
# If this is the same as the configuration option name, then
# continue to the next key in the list
if item.rstrip() == option.rstrip():
continue
if da is None:
if loaded.ca.items:
if item in loaded.ca.items:
desc = processComments(loaded.ca.items[item])
da = loaded[item]
elif isinstance(da, dict):
if item in da.keys()[0]:
commentTokens = da.ca.comment
if commentTokens is not None:
desc = processComments(commentTokens)
if da.ca.items:
if item in da.ca.items:
desc = desc + processComments(da.ca.items[item])
da = da[item]
newkey = newkey.lstrip('.')
# Filling Description Fields
if ("." in newkey):
plist = newkey.split('.')
da = None
for item in plist:
desc = ''
# If this is the same as the configuration option name, then
# continue to the next key in the list
if item.rstrip() == option.rstrip():
continue
if da is None:
if loaded.ca.items:
if item in loaded.ca.items:
desc = processComments(loaded.ca.items[item])
da = loaded[item]
elif isinstance(da, dict):
if item in da.keys()[0]:
commentTokens = da.ca.comment
if commentTokens is not None:
desc = processComments(commentTokens)
if da.ca.items:
if item in da.ca.items:
desc = desc + processComments(da.ca.items[item])
da = da[item]
else:
if item in da.keys():
da = da.get(item)
else:
if item in da.keys():
da = da.get(item)
else:
da = da.values()[0]
ValueStr = (' ').join(ValueList)
if ValueStr:
if (desc in ValueStr):
ValueStr= ValueStr.replace("#"+desc, "")
desc = desc.replace('`','')
desc = sanitizeValueStr(desc)
if desc.strip():
desc = '`' + desc.strip() + '`'
prdict[storekey].append("| `%s` | `%s` | %s |" % (newkey, ValueStr.rstrip(), desc))
desc = ''
da = da.values()[0]
ValueStr = (' ').join(ValueList)
if ValueStr:
if (desc in ValueStr):
ValueStr = ValueStr.replace("#" + desc, "")
desc = desc.replace('`', '')
desc = sanitizeValueStr(desc)
if desc.strip():
desc = '`' + desc.strip() + '`'
prdict[storekey].append("| `%s` | `%s` | %s |" % (newkey, ValueStr.rstrip(), desc))
desc = ''
key = newkey
newkey = ''
lineNum += 1
return ret_val
key = newkey
newkey = ''
lineNum += 1
return ret_val
def processComments(comments):
description = ''
@ -272,29 +277,29 @@ def processComments(comments):
pass
else:
# We want to avoid including commented out key: value pairs in the values.yaml as
# part of the description/comments. For example:
# part of the description/comments. For example:
# # minAvailable: 1
# # maxUnavailable: 1
# # - secretName: grafana-tls
# sessionAffinityEnabled: false
# We do not want the commented out key-value pairs (minAvailable,maxUnavailable, secretName)
# to be included as part of the description for 'sessionAffinityEnabled'
#
pattern = re.compile("#\s[-\s]*[\S]+:(?:\s(?!\S+:)\S+)*" )
#
pattern = re.compile(r"#\s[-\s]*[\S]+:(?:\s(?!\S+:)\S+)*")
groups = pattern.match(comment.value)
if groups:
description=''
description = ''
break
if comment.value.endswith('\n\n'):
description=''
description = ''
else:
if comment.value.rstrip() == '#':
continue
else:
description = description + comment.value.replace('`','').replace("#",'').rstrip()
description = description + comment.value.replace('`', '').replace("#", '').rstrip()
elif isinstance(c, yaml.Token):
description = description + c.value.rstrip().replace("#",'')
description = description + c.value.rstrip().replace("#", '')
return description
@ -303,18 +308,18 @@ def sanitizeValueStr(value):
# be escaped. For now just including the 'pipe' symbol appearing
# in the value of a configuration option.
# e.g: | `global.tracer.lightstep.secure` | `true # example: true\|false` | |
#
#
# Without escaping the 'pipe' character, it was interpreting it as the end/start
# of table column. Using the example above, without escaping the pipe symbol, it
# of table column. Using the example above, without escaping the pipe symbol, it
# was interpreting it as:
# | `global.tracer.lightstep.secure` | `true # example: true |false` | |
#
regex = re.compile("\|")
if value != None and regex.search(value) != None:
value = value.replace("|", "\|");
regex = re.compile(r"\|")
if value is not None and regex.search(value) is not None:
value = value.replace("|", r"\|")
return value
with open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIR), 'r') as f:
endReached = False
@ -334,16 +339,15 @@ with open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIR), 'r') as f:
# Print encoded string dictionary
for k, v in od.items():
print ("## `%s` options\n" % k)
print("## `%s` options\n" % k)
print '| Key | Default Value | Description |'
print '| --- | --- | --- |'
for value in v:
print ('%s' % (value))
print ('')
print('%s' % (value))
print('')
for d in data:
if "<!-- AUTO-GENERATED-END -->" in d:
endReached = True
if endReached:
print d

403
scripts/tablegen_diff.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python
# Copyright 2017,2018 Istio Authors. All Rights Reserved.
# Copyright Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -48,6 +48,7 @@ CONFIG_INDEX_DIR = "content/docs/reference/config/installation-options/index.md"
CONFIG_INDEX_DIFF_DIR = "content/docs/reference/config/installation-options-changes/index.md"
CONFIG_IGNORE_LIST = ["global.hub"]
def endOfTheList(context, lineNum, lastLineNum, totalNum):
flag = 0
valueList = []
@ -61,17 +62,17 @@ def endOfTheList(context, lineNum, lastLineNum, totalNum):
for nextLineNum in range(lineNum + 1, totalNum):
nextLine = context[nextLineNum]
if len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' in nextLine:
if len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' in nextLine:
if whitespaces >= (len(nextLine) - len(nextLine.lstrip())) / 2:
if flag == 0:
valueList.append(currentLine.split(':', 1)[1].strip())
return True, valueList
else:
return True, valueList
elif len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' not in nextLine and len(nextLine.strip()) != 0:
elif len(nextLine.lstrip()) != 0 and '#' != nextLine.lstrip()[0] and ':' not in nextLine and len(nextLine.strip()) != 0:
value = nextLine.replace(' ', '')
valueList.append(value.lstrip('-').strip())
flag += 1;
flag += 1
nextLineNum += 1
if lastLineNum == totalNum - 1 and len(currentLine.lstrip()) != 0 and '#' != currentLine.lstrip()[0]:
@ -79,20 +80,22 @@ def endOfTheList(context, lineNum, lastLineNum, totalNum):
return True, valueList
# ordered dictionary to store the configuration options for the subcomponents of Istio. This
# will be used to populate a new index.md
prdict = collections.defaultdict(list)
# ordered dictionary to store the differences of configuration options between the new
# ordered dictionary to store the differences of configuration options between the new
# index.md and the previous version (i.e, configurations options already listed in the index.md).
od_diff = collections.defaultdict(list)
od_diff_new = collections.defaultdict(list)
od_diff_removed = collections.defaultdict(list)
od_diff_unchanged = collections.defaultdict(list)
def decode_helm_yaml(s):
ret_val = ''
#
# Iterate through all the directories under /istio/install/kubernetes/heml/subcharts
# Iterate through all the directories under /istio/install/kubernetes/heml/subcharts
# and process the configuration options from the respective values.yaml. The
# configuration option name is the name of the directory that contains values.yaml.
# This name will be passed in to the the function process_helm_yaml
@ -114,166 +117,168 @@ def decode_helm_yaml(s):
return ret_val
def process_helm_yaml(values_yaml, option):
ret_val = ''
storekey = ''
desc = ''
newkey = ''
whitespaces = 0
flag = 0
lineNum = 0
newConfigList = []
loaded = None
ret_val = ''
storekey = ''
desc = ''
newkey = ''
whitespaces = 0
flag = 0
lineNum = 0
newConfigList = []
loaded = None
context = linecache.getlines(values_yaml)
totalNum = len(context)
lastLineNum = 0
key = option
count = 0
with open(values_yaml, 'r') as f_v:
d_v = f_v.read()
loaded = yaml.round_trip_load(d_v)
context = linecache.getlines(values_yaml)
totalNum = len(context)
lastLineNum = 0
key = option
for lineNum in range(0, totalNum):
if context[lineNum].strip().startswith('- '):
pass
elif '#' in context[lineNum] and '#' == context[lineNum].lstrip()[0]:
if "Description: " in context[lineNum]:
desc = context[lineNum].strip()
elif ':' in context[lineNum] and '#' != context[lineNum].lstrip()[0]:
lastLineNum = lineNum
if flag == 1:
whitespaces = (len(context[lineNum]) - len(context[lineNum].lstrip())) / 2
periods = key.count('.')
if (option == ''):
while (whitespaces <= periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
else:
while (whitespaces < periods) :
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
flag = 0
count = 0
with open(values_yaml, 'r') as f_v:
d_v = f_v.read()
loaded = yaml.round_trip_load(d_v)
key = key + '.' + context[lineNum].split(':', 1)[0].strip()
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
if isEnd == True:
flag = 1;
for lineNum in range(0, totalNum):
if context[lineNum].strip().startswith('- '):
pass
elif '#' in context[lineNum] and '#' == context[lineNum].lstrip()[0]:
if "Description: " in context[lineNum]:
desc = context[lineNum].strip()
elif ':' in context[lineNum] and '#' != context[lineNum].lstrip()[0]:
lastLineNum = lineNum
if flag == 1:
whitespaces = (len(context[lineNum]) - len(context[lineNum].lstrip())) / 2
periods = key.count('.')
if (option == ''):
while (whitespaces <= periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
else:
while (whitespaces < periods):
key = key.rstrip(string.ascii_letters[::-1] + string.digits + '_' + '-' + '/').rstrip('.')
whitespaces += 1
flag = 0
storekey = key
sk = storekey.split('.', 2)
if len(sk) > 1:
storekey = '.'.join(sk[:1]).lstrip('.')
else:
storekey = '.'.join(sk[:0]).lstrip('.')
key = key + '.' + context[lineNum].split(':', 1)[0].strip()
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
if isEnd:
flag = 1
#
# If we are processing the configurations options within the values.yaml under istio,
# if the options have already been processed (from the subcharts directory), then we
# do not want to process it again. If the configuration option has not been processed
# before, then it is a new configuration option which needs to be processed (for e.g,
# global, istiocoredns)
#
# option == '' - This condition means that we are looking at the values.yaml under the
# istio directory. Hence, the configuration option names will be inside
# the values.yaml file. (On the other hand, for the values.yaml file under
# the subcharts directory, we get the name of the configuration option
# from the name of the directories under the subcharts directory.)
# newConfigList - This list is used to track configuration options in istio/values.yaml
# that haven't been processed before (or that does not have a corresponding
# directory under subcharts directory with values.yaml. E.g: global,
# istiocoredns)
#
# This first condition checks that if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has not already been processed (this
# conditions: "prdict.get(storekey) != None and (storekey in newConfigList)" together
# makes sure that the condition where some parameters for a new configuration option like
# 'global' has been processed and entered into the dictionary 'prdict' is still processed
# because it is in the newConfigList. If a configuration option was processed from
# the values.yaml under the subcharts directory, it will not be in the newConfigList.
# subcharts directory), then go ahead and process the parameters for this option.
#
if option == '' and prdict.get(storekey) != None and (storekey in newConfigList):
pass
#
# This second condition checks if this is the values.yaml file under istio directory, and
# the configuration option to process (storekey) has not been processed (this could
# happen the first time we read a configuration option from the istio/values.yaml file),
# then add this configuration option to the newConfigList to mark it as an option that
# needs to be processed.
#
elif option == '' and prdict.get(storekey) == None:
newConfigList.append(storekey)
#
# This third condition checks if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has already been processed and if
# this is not a new configuration option, (this could happen if we have already
# processed the corresponding values.yaml under the subcharts directory), then ignore
# this configuration option and do not process the values in this file.
#
elif option == '' and prdict.get(storekey) != None:
continue
storekey = key
sk = storekey.split('.', 2)
if len(sk) > 1:
storekey = '.'.join(sk[:1]).lstrip('.')
else:
storekey = '.'.join(sk[:0]).lstrip('.')
if len(context[lastLineNum].lstrip()) != 0 and '#' != context[lastLineNum].lstrip()[0]:
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
#
# If we are processing the configurations options within the values.yaml under istio,
# if the options have already been processed (from the subcharts directory), then we
# do not want to process it again. If the configuration option has not been processed
# before, then it is a new configuration option which needs to be processed (for e.g,
# global, istiocoredns)
#
# option == '' - This condition means that we are looking at the values.yaml under the
# istio directory. Hence, the configuration option names will be inside
# the values.yaml file. (On the other hand, for the values.yaml file under
# the subcharts directory, we get the name of the configuration option
# from the name of the directories under the subcharts directory.)
# newConfigList - This list is used to track configuration options in istio/values.yaml
# that haven't been processed before (or that does not have a corresponding
# directory under subcharts directory with values.yaml. E.g: global,
# istiocoredns)
#
# This first condition checks that if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has not already been processed (this
# conditions: "prdict.get(storekey) != None and (storekey in newConfigList)" together
# makes sure that the condition where some parameters for a new configuration option like
# 'global' has been processed and entered into the dictionary 'prdict' is still processed
# because it is in the newConfigList. If a configuration option was processed from
# the values.yaml under the subcharts directory, it will not be in the newConfigList.
# subcharts directory), then go ahead and process the parameters for this option.
#
if option == '' and prdict.get(storekey) is not None and (storekey in newConfigList):
pass
#
# This second condition checks if this is the values.yaml file under istio directory, and
# the configuration option to process (storekey) has not been processed (this could
# happen the first time we read a configuration option from the istio/values.yaml file),
# then add this configuration option to the newConfigList to mark it as an option that
# needs to be processed.
#
elif option == '' and prdict.get(storekey) is None:
newConfigList.append(storekey)
#
# This third condition checks if this is the values.yaml file under istio directory,
# and the configuration option to process (storekey) has already been processed and if
# this is not a new configuration option, (this could happen if we have already
# processed the corresponding values.yaml under the subcharts directory), then ignore
# this configuration option and do not process the values in this file.
#
elif option == '' and prdict.get(storekey) is not None:
continue
if (isEnd == True):
flag = 1
keysplit = key.split('.')
for kv in keysplit:
if kv != '':
newkey = newkey + '.' + kv
if len(context[lastLineNum].lstrip()) != 0 and '#' != context[lastLineNum].lstrip()[0]:
isEnd, ValueList = endOfTheList(context, lineNum, lastLineNum, totalNum)
newkey = newkey.lstrip('.')
if (isEnd):
flag = 1
keysplit = key.split('.')
for kv in keysplit:
if kv != '':
newkey = newkey + '.' + kv
# Filling Description Fields
if ( "." in newkey):
plist = newkey.split('.')
da = None
for item in plist:
desc = ''
# If this is the same as the configuration option name, then
# continue to the next key in the list
if item.rstrip() == option.rstrip():
continue
if da is None:
if loaded.ca.items:
if item in loaded.ca.items:
desc = processComments(loaded.ca.items[item])
da = loaded[item]
elif isinstance(da, dict):
if item in da.keys()[0]:
commentTokens = da.ca.comment
if commentTokens is not None:
desc = processComments(commentTokens)
if da.ca.items:
if item in da.ca.items:
desc = desc + processComments(da.ca.items[item])
da = da[item]
newkey = newkey.lstrip('.')
# Filling Description Fields
if ("." in newkey):
plist = newkey.split('.')
da = None
for item in plist:
desc = ''
# If this is the same as the configuration option name, then
# continue to the next key in the list
if item.rstrip() == option.rstrip():
continue
if da is None:
if loaded.ca.items:
if item in loaded.ca.items:
desc = processComments(loaded.ca.items[item])
da = loaded[item]
elif isinstance(da, dict):
if item in da.keys()[0]:
commentTokens = da.ca.comment
if commentTokens is not None:
desc = processComments(commentTokens)
if da.ca.items:
if item in da.ca.items:
desc = desc + processComments(da.ca.items[item])
da = da[item]
else:
if item in da.keys():
da = da.get(item)
else:
if item in da.keys():
da = da.get(item)
else:
da = da.values()[0]
ValueStr = (' ').join(ValueList)
if ValueStr:
if (desc in ValueStr):
ValueStr= ValueStr.replace("#"+desc, "")
desc = desc.replace('`','')
desc = sanitizeValueStr(desc)
if desc.strip():
desc = '`' + desc.strip() + '`'
prdict[storekey].append("| `%s` | `%s` | %s |" % (newkey, ValueStr.rstrip(), desc))
desc = ''
da = da.values()[0]
ValueStr = (' ').join(ValueList)
if ValueStr:
if (desc in ValueStr):
ValueStr = ValueStr.replace("#" + desc, "")
desc = desc.replace('`', '')
desc = sanitizeValueStr(desc)
if desc.strip():
desc = '`' + desc.strip() + '`'
prdict[storekey].append("| `%s` | `%s` | %s |" % (newkey, ValueStr.rstrip(), desc))
desc = ''
key = newkey
newkey = ''
lineNum += 1
return ret_val
key = newkey
newkey = ''
lineNum += 1
return ret_val
def processComments(comments):
description = ''
@ -286,29 +291,29 @@ def processComments(comments):
pass
else:
# We want to avoid including commented out key: value pairs in the values.yaml as
# part of the description/comments. For example:
# part of the description/comments. For example:
# # minAvailable: 1
# # maxUnavailable: 1
# # - secretName: grafana-tls
# sessionAffinityEnabled: false
# We do not want the commented out key-value pairs (minAvailable,maxUnavailable, secretName)
# to be included as part of the description for 'sessionAffinityEnabled'
#
pattern = re.compile("#\s[-\s]*[\S]+:(?:\s(?!\S+:)\S+)*" )
#
pattern = re.compile(r"#\s[-\s]*[\S]+:(?:\s(?!\S+:)\S+)*")
groups = pattern.match(comment.value)
if groups:
description=''
description = ''
break
if comment.value.endswith('\n\n'):
description=''
description = ''
else:
if comment.value.rstrip() == '#':
continue
else:
description = description + comment.value.replace('`','').replace("#",'').rstrip()
description = description + comment.value.replace('`', '').replace("#", '').rstrip()
elif isinstance(c, yaml.Token):
description = description + c.value.rstrip().replace("#",'')
description = description + c.value.rstrip().replace("#", '')
return description
@ -317,15 +322,15 @@ def sanitizeValueStr(value):
# be escaped. For now just including the 'pipe' symbol appearing
# in the value of a configuration option.
# e.g: | `global.tracer.lightstep.secure` | `true # example: true\|false` | |
#
#
# Without escaping the 'pipe' character, it was interpreting it as the end/start
# of table column. Using the example above, without escaping the pipe symbol, it
# of table column. Using the example above, without escaping the pipe symbol, it
# was interpreting it as:
# | `global.tracer.lightstep.secure` | `true # example: true |false` | |
#
regex = re.compile("\|")
if value != None and regex.search(value) != None:
value = value.replace("|", "\|");
regex = re.compile(r"\|")
if value is not None and regex.search(value) is not None:
value = value.replace("|", r"\|")
return value
# Compares the configuration option value from the newly discovered set of values (stored
@ -334,25 +339,27 @@ def sanitizeValueStr(value):
# there are any differences, we will store the differences (will track differences for key,
# value and description of a configuration option) in the 'od_diff' dictionary. The values
# stored in this dictionary will later be written to CONFIG_INDEX_DIFF_DIR.
#
#
# The difference between the configuration option values is stored in the CONFIG_INDEX_DIFF_DIR
# in the format:
# | KEY | OLD DEFAULT VALUE | NEW DEFAULT VALUE | OLD DESCRIPTION | NEW DESCRIPTION |
# | ------ | ------------ | ------------ | ------------ | ------------ |
# | Key | oldValue | newValue | oldDesc | newDesc |
#
#
# If a configuration option is present only in the latest version, then the oldKey, oldValue
# and oldDescription will be represented as 'n/a' (vice-versa applies to newKey, newValue and
# newDescription).
# newDescription).
#
# oValue - configuration option from the existing index.md
# oValue - configuration option from the existing index.md
# nValue - configuration option from the current processing of configuration options to be
# stored in a new version of index.md
# k - istio component name for which these configuration options are being processed. This is
# used to populate the contents of 'od_diff' dictionary.
#
def compareValues(oValue, nValue, k):
# oValue and nVAlue contains configuration option in the format:
# oValue and nVAlue contains configuration option in the format:
# '| `<Key>` | `<Value>` | `<Description>` |
# This needs to be split in order to get the Key, Value and Description values to compare.
oldKey = ''
@ -364,14 +371,14 @@ def compareValues(oValue, nValue, k):
newDesc = ''
key = None
if nValue is not None:
groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", nValue.strip())
if nValue is not None:
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \| (.*) |", nValue.strip())
if groups:
newKey = groups.group(1)
newValue = groups.group(2)
newDesc = groups.group(3)
if oValue is not None and nValue is not None:
if len(oValue) == 1:
item = oValue[0]
@ -381,7 +388,7 @@ def compareValues(oValue, nValue, k):
oValue.remove(item)
od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
else:
groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
if groups:
oldKey = groups.group(1)
oldValue = groups.group(2)
@ -389,7 +396,7 @@ def compareValues(oValue, nValue, k):
key = oldKey
if oldKey in CONFIG_IGNORE_LIST:
oValue.remove(item)
oValue.remove(item)
return key
if oldValue != newValue:
@ -399,7 +406,7 @@ def compareValues(oValue, nValue, k):
newValue = 'n/a'
if oldDesc.strip() != newDesc.strip():
if (newDesc == None or newDesc == '') and (oldDesc is None or oldDesc == ''):
if (newDesc is None or newDesc == '') and (oldDesc is None or oldDesc == ''):
pass
if oldDesc is None:
oldDesc = 'n/a'
@ -408,11 +415,11 @@ def compareValues(oValue, nValue, k):
oValue.remove(item)
od_diff[k].append("| `%s` | `%s` | `%s` | %s | %s |" % (newKey, oldValue.rstrip(), newValue.rstrip(), oldDesc, newDesc))
else:
# This is the case where values are the same but descriptions are different. Right now, there is nothing more to do since
# we do not care about displaying values that haven't changed between releases.
# This is the case where values are the same but descriptions are different. Right now, there is nothing more to do since
# we do not care about displaying values that haven't changed between releases.
oValue.remove(item)
#od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
#od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
else:
foundItem = 'false'
for item in oValue:
@ -423,7 +430,7 @@ def compareValues(oValue, nValue, k):
foundItem = 'true'
break
else:
groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
if groups:
oldKey = groups.group(1)
oldValue = groups.group(2)
@ -441,10 +448,10 @@ def compareValues(oValue, nValue, k):
od_diff_new[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
elif oValue is None:
key = newKey
od_diff_new[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
od_diff_new[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
elif nValue is None:
for item in oValue:
groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
if groups:
oldKey = groups.group(1)
oldValue = groups.group(2)
@ -455,10 +462,12 @@ def compareValues(oValue, nValue, k):
return key
#
#
# Get the previous release number so that we can retrieve the index.md for that
# release. The release branches are tagged in the following format: release-<number>
#
def getPreviousRelease():
req = requests.get("https://api.github.com/repos/istio/istio/branches")
jsonData = req.json()
@ -475,8 +484,10 @@ def getPreviousRelease():
#
# Get the index.md for the previous release.
#
def getContentFromPreviousRelease(releaseName):
istio_url = 'https://raw.githubusercontent.com/istio/istio.io/release-' + releaseName +'/content/docs/reference/config/installation-options/index.md'
istio_url = 'https://raw.githubusercontent.com/istio/istio.io/release-' + releaseName + '/content/docs/reference/config/installation-options/index.md'
req = requests.get(istio_url)
content = req.text
indexMap = collections.defaultdict(list)
@ -487,7 +498,7 @@ def getContentFromPreviousRelease(releaseName):
data = content.split('\n')
for d in data:
if d.rstrip() != '' and d != '| Key | Default Value | Description |' and d != '| --- | --- | --- |' and d[0:1] == '|' and d[-1] == '|':
groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", d.strip())
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \| (.*) |", d.strip())
if groups:
key = groups.group(1)
if key in indexMap:
@ -497,6 +508,7 @@ def getContentFromPreviousRelease(releaseName):
indexMap[key].append(d.strip())
return indexMap
def writeVersionDiffs(index_diff_file):
meta = ""
@ -525,7 +537,7 @@ def writeVersionDiffs(index_diff_file):
index_diff_file.write('\n## Modified configuration options\n')
for k, v in od_diff.items():
index_diff_file.write("\n### Modified `%s` key/value pairs\n\n" % k)
index_diff_file.write("\n### Modified `%s` key/value pairs\n\n" % k)
index_diff_file.write('| Key | Old Default Value | New Default Value | Old Description | New Description |\n')
index_diff_file.write('| --- | --- | --- | --- | --- |\n')
@ -557,11 +569,12 @@ def writeVersionDiffs(index_diff_file):
index_diff_file.write("\n<!-- AUTO-GENERATED-END -->\n")
index_diff_file.truncate()
with open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIR), 'r') as f:
endReached = False
key = ''
# A list used to track the configuration options that has been compared and processed when going
# through the configurations processed in the latest version
# through the configurations processed in the latest version
indexList = []
previousRelease = getPreviousRelease()
indexMap = getContentFromPreviousRelease(previousRelease)
@ -579,19 +592,19 @@ with open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIR), 'r') as f:
for value in v:
# Compare configuration option values from the latest version
# with the older version.
groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", value.strip())
groups = re.search(r"\| \`(.*)\` \| \`(.*)\` \| (.*) |", value.strip())
if groups:
key = groups.group(1)
indexValue = indexMap.get(key)
indexList.append(compareValues(indexValue, value, k))
# We want to include any configuration options that was discovered in
# the older version but not available in the current version
for k in indexMap.keys():
key = k.split('.')[0]
indexList.append(compareValues(indexMap.get(k), None, key))
indexList.append(compareValues(indexMap.get(k), None, key))
# This index.md file is used to track the differences of configuration
# option values between the current and previous release. All the
# differences in configuration option values between the current