From 070bfee1303eea324f09816fa3764c67da521b6d Mon Sep 17 00:00:00 2001 From: Max Jonas Werner Date: Tue, 4 Jul 2023 12:02:29 +0200 Subject: [PATCH] Generalize import-flux2-assets.sh script This is the first step towards supporting multiple versions of Flux to be documented in parallel. The script has been condensed to only require a single input parameter denoting the desired Flux major and minor version for which to build the documentation. This parameter is read from the environment variable `BRANCH` (which Netlify) automatically provides and has to be either `main` or have the format `vX-Y` which `X` denoting the major and `Y` denoting the minor version desired to build the docs for. The intent is to have branches following this naming pattern that Netlify will directly deploy under the URL vX-Y.fluxcd.io. The changed import-flux2-assets.sh script now conducts the following steps in order to generate the docs for a specific version: 1. Find most recent patch version of Flux matching the given `BRANCH`. 2. For each controller fetch the `kustomization.yaml` from https://github.com/fluxcd/flux2/blob/FLUX_VERSION/manifests/bases/CONTROLLER/kustomization.yaml 3. From there fetch the `*.crds.yaml` file and the controller version. 4. For each CRD group version fetch the API docs from the controller repository as well as the spec docs. Some special cases remain, e.g. for IRC/IAC which are supposed to be documented under the same main navigation entry and for NC's `Event` type which is not a CRD. refs #1534 Co-authored-by: Kingdon Barrett Signed-off-by: Max Jonas Werner Signed-off-by: Kingdon Barrett --- .gitignore | 3 + Makefile | 21 ++- README.md | 13 +- content/en/flux/components/helm/.gitignore | 2 +- content/en/flux/components/helm/api/_index.md | 5 + content/en/flux/components/helm/options.md | 2 +- content/en/flux/components/image/.gitignore | 4 +- .../components/image/automation-api/_index.md | 5 + content/en/flux/components/image/options.md | 2 +- .../components/image/reflector-api/_index.md | 5 + .../en/flux/components/kustomize/.gitignore | 2 +- .../flux/components/notification/.gitignore | 8 +- hack/import-flux2-assets.sh | 160 ++++++++++-------- 13 files changed, 147 insertions(+), 85 deletions(-) create mode 100644 content/en/flux/components/helm/api/_index.md create mode 100644 content/en/flux/components/image/automation-api/_index.md create mode 100644 content/en/flux/components/image/reflector-api/_index.md diff --git a/.gitignore b/.gitignore index 6b8416a2..0d556ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ static/install.sh # local GH token, used for some scripts .gh-token + +# build directories +/bin diff --git a/Makefile b/Makefile index 8d217a0e..fd5bcadf 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,12 @@ HUGO_BIND_ADDRESS ?= 127.0.0.1 BUILDER_CLI := docker # BUILDER_CLI := okteto LYCHEE_IMAGE_NAME ?= lycheeverse/lychee:202105190720247e4977 +YQ_VERSION ?= v4.34.2 +BRANCH ?= main + +REPO_ROOT := $(shell git rev-parse --show-toplevel) +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +BIN_DIR := $(REPO_ROOT)/bin help: ## Display this help menu. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -20,11 +26,20 @@ help: ## Display this help menu. prereqs: npm install + python3 -m pip install -r requirements.txt -gen-content: ## Generates content from external sources. +.PHONY: yq +yq: $(BIN_DIR)/yq-$(YQ_VERSION) + +$(BIN_DIR)/yq-$(YQ_VERSION): + mkdir -p $(BIN_DIR) && wget https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(shell go env GOARCH) -O $(BIN_DIR)/yq-$(YQ_VERSION) && \ + chmod +x $(BIN_DIR)/yq-$(YQ_VERSION) && \ + cp $(BIN_DIR)/yq-$(YQ_VERSION) $(BIN_DIR)/yq + +gen-content: yq ## Generates content from external sources. hack/gen-content.py hack/import-calendar.py - hack/import-flux2-assets.sh + PATH=$(BIN_DIR):$(PATH) BRANCH=$(BRANCH) hack/import-flux2-assets.sh serve: gen-content prereqs ## Spawns a development server. hugo server \ @@ -63,7 +78,7 @@ docker-preview: docker-serve .PHONY: docker-serve docker-serve: docker run -v $(shell pwd):/site -p 1313:1313 -it $(SUPPORT_IMAGE_NAME) \ - make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 + make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 GITHUB_TOKEN=$(GITHUB_TOKEN) BRANCH=$(BRANCH) .PHONY: docker-push docker-push-hugo docker-push-support docker-push: docker-push-hugo docker-push-support diff --git a/README.md b/README.md index 516dc7bf..cf4f848f 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,6 @@ In order to run the Flux site locally, you need to install: Once those tools are installed, fetch the assets necessary to run the site: -```cli -npm install -python3 -m pip install -r requirements.txt -``` - Then run the site in "server" mode: ```cli @@ -45,13 +40,19 @@ make serve Navigate to to see the site running in your browser. As you make updates to the site, the browser will immediately update to reflect those changes. +> **Note**: Sometimes you need to clear the output of previous builds, e.g. when the structure of imported documentation has changed. If you see duplicate pages being served, run +> ``` +> git clean -fx -- content/ +> ``` +> and then run `make serve` again. The duplicates will then be gone. + ## Publishing the site The Flux website is published automatically by [Netlify](https://netlify.com) when changes are pushed to the `main` branch. The site does not need to be published manually. ### Preview builds -When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **deploy/netlify** link in the pull request window. +When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **Details** link of the **netlify/fluxcd/deploy-preview** check at the bottom of the pull request page. ## Local Development (docker) diff --git a/content/en/flux/components/helm/.gitignore b/content/en/flux/components/helm/.gitignore index d57ee8b5..5264c36c 100644 --- a/content/en/flux/components/helm/.gitignore +++ b/content/en/flux/components/helm/.gitignore @@ -1,2 +1,2 @@ -api.md +api/v*.md helmreleases.md diff --git a/content/en/flux/components/helm/api/_index.md b/content/en/flux/components/helm/api/_index.md new file mode 100644 index 00000000..ea8b9134 --- /dev/null +++ b/content/en/flux/components/helm/api/_index.md @@ -0,0 +1,5 @@ +--- +title: API Reference +linkTitle: API Reference +weight: 1000 +--- diff --git a/content/en/flux/components/helm/options.md b/content/en/flux/components/helm/options.md index d781c61d..71c4d7fb 100644 --- a/content/en/flux/components/helm/options.md +++ b/content/en/flux/components/helm/options.md @@ -2,7 +2,7 @@ title: Controller Options linkTitle: Controller Options description: "Controller command flags and defaults." -weight: 1000 +weight: 1 --- To customise the controller options at install time, diff --git a/content/en/flux/components/image/.gitignore b/content/en/flux/components/image/.gitignore index 84061d40..3af01be5 100644 --- a/content/en/flux/components/image/.gitignore +++ b/content/en/flux/components/image/.gitignore @@ -1,5 +1,5 @@ -automation-api.md +automation-api/v*.md +reflector-api/v*.md imagepolicies.md imagerepositories.md imageupdateautomations.md -reflector-api.md diff --git a/content/en/flux/components/image/automation-api/_index.md b/content/en/flux/components/image/automation-api/_index.md new file mode 100644 index 00000000..9a653328 --- /dev/null +++ b/content/en/flux/components/image/automation-api/_index.md @@ -0,0 +1,5 @@ +--- +title: Image Update Automation API Reference +linkTitle: Image Update Automation API Reference +weight: 70 +--- diff --git a/content/en/flux/components/image/options.md b/content/en/flux/components/image/options.md index 9862735b..710d6d9e 100644 --- a/content/en/flux/components/image/options.md +++ b/content/en/flux/components/image/options.md @@ -2,7 +2,7 @@ title: Controller Options linkTitle: Controller Options description: "Controller command flags and defaults." -weight: 1000 +weight: 1 --- To customise the controller options at install time, diff --git a/content/en/flux/components/image/reflector-api/_index.md b/content/en/flux/components/image/reflector-api/_index.md new file mode 100644 index 00000000..b9765032 --- /dev/null +++ b/content/en/flux/components/image/reflector-api/_index.md @@ -0,0 +1,5 @@ +--- +title: Image Reflector API Reference +linkTitle: Image Reflector API Reference +weight: 70 +--- diff --git a/content/en/flux/components/kustomize/.gitignore b/content/en/flux/components/kustomize/.gitignore index db062c04..52199d1c 100644 --- a/content/en/flux/components/kustomize/.gitignore +++ b/content/en/flux/components/kustomize/.gitignore @@ -1,2 +1,2 @@ api/v*.md -kustomization.md +kustomizations.md diff --git a/content/en/flux/components/notification/.gitignore b/content/en/flux/components/notification/.gitignore index 772e150e..d523d6fc 100644 --- a/content/en/flux/components/notification/.gitignore +++ b/content/en/flux/components/notification/.gitignore @@ -1,5 +1,5 @@ -alert.md +alerts.md api/v*.md -event.md -provider.md -receiver.md +events.md +providers.md +receivers.md diff --git a/hack/import-flux2-assets.sh b/hack/import-flux2-assets.sh index e937c9a3..5a2d8aef 100755 --- a/hack/import-flux2-assets.sh +++ b/hack/import-flux2-assets.sh @@ -1,9 +1,11 @@ -#!/bin/sh -x +#!/usr/bin/env bash + +set -euxo pipefail COMPONENTS_DIR="content/en/flux/components" FLUX_DIR="content/en/flux/cmd" -if [ -z "${GITHUB_USER}" ]; then +if [ -z "${GITHUB_USER:-}" ]; then GITHUB_USER=fluxcdbot fi @@ -19,7 +21,7 @@ fatal() { # Set os, fatal if operating system not supported setup_verify_os() { - if [ -z "${OS}" ]; then + if [ -z "${OS:-}" ]; then OS=$(uname) fi case ${OS} in @@ -36,7 +38,7 @@ setup_verify_os() { # Set arch, fatal if architecture not supported setup_verify_arch() { - if [ -z "${ARCH}" ]; then + if [ -z "${ARCH:-}" ]; then ARCH=$(uname -m) fi case ${ARCH} in @@ -58,22 +60,10 @@ setup_verify_arch() { } -controller_version() { - url=https://api.github.com/repos/fluxcd/$1/releases - out=$(curl -w "%{http_code}" -o /tmp/releases -u "$GITHUB_USER:$GITHUB_TOKEN" -s "$url") - if [ "${out}" -ne "200" ] ; then - echo "error fetching releases from $url: $out" - cat /tmp/releases - exit 1 - fi - - jq -r '.[] | .tag_name' < /tmp/releases | sed 's/.*\///' | awk '{ if ($1 ~ /-/) print; else print $0"_" ; }' | sort -rV | sed 's/_$//' | head -n1 -} - gen_crd_doc() { URL="$1" DEST="$2" - HUGETABLE="$3" + HUGETABLE="${3:-}" TMP="$(mktemp)" curl -u "$GITHUB_USER:$GITHUB_TOKEN" -# -Lf "$URL" > "$TMP" @@ -85,7 +75,7 @@ gen_crd_doc() { # 3. While others go # Here is the title you're looking for... # - FIRST_LINE="$(grep -vE "^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2)" + WEIGHT="$(grep -E '^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2 || true)" if [ -z "${WEIGHT}" ] ; then WEIGHT=0 fi @@ -119,55 +109,47 @@ gen_crd_doc() { fi } -{ - # source-controller CRDs - SOURCE_VER="$(controller_version source-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1/source.md" "$COMPONENTS_DIR/source/api/v1.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1beta2/source.md" "$COMPONENTS_DIR/source/api/v1beta2.md" "HUGETABLE" - SOURCE_CRD_VER="v1" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/gitrepositories.md" "$COMPONENTS_DIR/source/gitrepositories.md" - SOURCE_CRD_VER="v1beta2" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/ocirepositories.md" "$COMPONENTS_DIR/source/ocirepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmrepositories.md" "$COMPONENTS_DIR/source/helmrepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmcharts.md" "$COMPONENTS_DIR/source/helmcharts.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/buckets.md" "$COMPONENTS_DIR/source/buckets.md" +function all_versions { + for crd in $1 ; do + echo "${crd##*,}" + done | sort | uniq } -{ - # kustomize-controller CRDs - KUSTOMIZE_VER="$(controller_version kustomize-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/api/v1/kustomize.md" "$COMPONENTS_DIR/kustomize/api/v1.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/spec/v1/kustomization.md" "$COMPONENTS_DIR/kustomize/kustomization.md" -} +function gen_ctrl_docs { + flux_version=${1} + ctrl=${2} + ctrl_short=${ctrl%%-controller} + ctrl_out=${ctrl%%-*} # this is necessary to collect IAC and IRC together in the "image" folder -{ - # helm-controller CRDs - HELM_VER="$(controller_version helm-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/api/helmrelease.md" "$COMPONENTS_DIR/helm/api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/spec/v2beta1/helmreleases.md" "$COMPONENTS_DIR/helm/helmreleases.md" -} + api_out="api" + if [ "${ctrl_short}" = "image-reflector" ] ; then + api_out="reflector-api" + fi + if [ "${ctrl_short}" = "image-automation" ] ; then + api_out="automation-api" + fi -{ - # notification-controller CRDs - NOTIFICATION_VER="$(controller_version notification-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1/notification.md" "$COMPONENTS_DIR/notification/api/v1.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1beta2/notification.md" "$COMPONENTS_DIR/notification/api/v1beta2.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/notification/event.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/alerts.md" "$COMPONENTS_DIR/notification/alert.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/providers.md" "$COMPONENTS_DIR/notification/provider.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1/receivers.md" "$COMPONENTS_DIR/notification/receiver.md" -} + ks_url=$(curl -sL "https://raw.githubusercontent.com/fluxcd/flux2/${flux_version}/manifests/bases/${ctrl}/kustomization.yaml" | yq '.resources[]|select(. == "*crds.yaml*")') + ctrl_version=$(echo "${ks_url}" | cut -d/ -f8) -{ - # image-*-controller CRDs; these use the same API group - IMG_REFL_VER="$(controller_version image-reflector-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/api/image-reflector.md" "$COMPONENTS_DIR/image/reflector-api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagerepositories.md" "$COMPONENTS_DIR/image/imagerepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagepolicies.md" "$COMPONENTS_DIR/image/imagepolicies.md" + crds=$(curl -sL "${ks_url}" | yq ea '[[.metadata.name, .spec.versions[] | select(.storage == "true").name]]' -o csv) + for api_version in $(all_versions "${crds}") ; do + doc_url=https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/api/${api_version}/${ctrl_short}.md + gen_crd_doc "${doc_url}" "$COMPONENTS_DIR/${ctrl_out}/${api_out}/${api_version}.md" "HUGETABLE" - IMG_AUTO_VER="$(controller_version image-automation-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/api/image-automation.md" "$COMPONENTS_DIR/image/automation-api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/spec/v1beta1/imageupdateautomations.md" "$COMPONENTS_DIR/image/imageupdateautomations.md" + done + + for crd in ${crds} ; do + name=${crd%%,*} + name=${name%%.*} + version=${crd##*,} + gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/${version}/${name}.md" "$COMPONENTS_DIR/${ctrl_out}/${name}.md" + done + + # special case: n-c's `Events` type is not a CRD but needs to be documented, too. + if [ "${ctrl}" = "notification-controller" ] ; then + gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/${ctrl_out}/events.md" + fi } { @@ -176,11 +158,31 @@ gen_crd_doc() { setup_verify_arch TMP="$(mktemp -d)" - TMP_METADATA="$TMP/flux.json" TMP_BIN="$TMP/flux.tar.gz" - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_METADATA}" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases/latest" - VERSION_FLUX=$(grep '"tag_name":' "${TMP_METADATA}" | sed -E 's/.*"([^"]+)".*/\1/' | cut -c 2-) + if [ -z "${BRANCH:-}" ] ; then + fatal "BRANCH environment variable not set" + fi + + if [[ "${BRANCH}" =~ ^pull/[[:digit:]]*/head$ ]] ; then + BRANCH=$(curl -sSfL "https://api.github.com/repos/fluxcd/website/pulls/$(echo ${BRANCH}|cut -d/ -f2)" | jq .base.ref -r) + fi + + VERSION_FLUX= + for tag in $(curl -u "$GITHUB_USER:$GITHUB_TOKEN" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases" | jq .[].tag_name -r) ; do + if [ "${BRANCH}" = "main" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + if [ "${tag%.*}" = "${BRANCH/-/.}" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + done + + if [ -z "${VERSION_FLUX}" ] ; then + fatal "No Flux version found matching branch '${BRANCH}'" + fi curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz" tar xfz "${TMP_BIN}" -C "${TMP}" @@ -191,11 +193,37 @@ gen_crd_doc() { rm -rf "$TMP" } +{ + # source-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "source-controller" +} + +{ + # kustomize-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "kustomize-controller" +} + +{ + # helm-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "helm-controller" +} + +{ + # notification-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "notification-controller" +} + +{ + # image-*-controller CRDs; these use the same API group + gen_ctrl_docs "v${VERSION_FLUX}" "image-reflector-controller" + gen_ctrl_docs "v${VERSION_FLUX}" "image-automation-controller" +} + { # provide Flux install script if [ ! -d static ]; then mkdir static fi - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf https://raw.githubusercontent.com/fluxcd/flux2/main/install/flux.sh -o static/install.sh + curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf "https://raw.githubusercontent.com/fluxcd/flux2/v${VERSION_FLUX}/install/flux.sh" -o static/install.sh }