diff --git a/.gitignore b/.gitignore index 6b8416a2..0d556ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ static/install.sh # local GH token, used for some scripts .gh-token + +# build directories +/bin diff --git a/Makefile b/Makefile index 8d217a0e..fd5bcadf 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,12 @@ HUGO_BIND_ADDRESS ?= 127.0.0.1 BUILDER_CLI := docker # BUILDER_CLI := okteto LYCHEE_IMAGE_NAME ?= lycheeverse/lychee:202105190720247e4977 +YQ_VERSION ?= v4.34.2 +BRANCH ?= main + +REPO_ROOT := $(shell git rev-parse --show-toplevel) +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +BIN_DIR := $(REPO_ROOT)/bin help: ## Display this help menu. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -20,11 +26,20 @@ help: ## Display this help menu. prereqs: npm install + python3 -m pip install -r requirements.txt -gen-content: ## Generates content from external sources. +.PHONY: yq +yq: $(BIN_DIR)/yq-$(YQ_VERSION) + +$(BIN_DIR)/yq-$(YQ_VERSION): + mkdir -p $(BIN_DIR) && wget https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(shell go env GOARCH) -O $(BIN_DIR)/yq-$(YQ_VERSION) && \ + chmod +x $(BIN_DIR)/yq-$(YQ_VERSION) && \ + cp $(BIN_DIR)/yq-$(YQ_VERSION) $(BIN_DIR)/yq + +gen-content: yq ## Generates content from external sources. hack/gen-content.py hack/import-calendar.py - hack/import-flux2-assets.sh + PATH=$(BIN_DIR):$(PATH) BRANCH=$(BRANCH) hack/import-flux2-assets.sh serve: gen-content prereqs ## Spawns a development server. hugo server \ @@ -63,7 +78,7 @@ docker-preview: docker-serve .PHONY: docker-serve docker-serve: docker run -v $(shell pwd):/site -p 1313:1313 -it $(SUPPORT_IMAGE_NAME) \ - make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 + make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 GITHUB_TOKEN=$(GITHUB_TOKEN) BRANCH=$(BRANCH) .PHONY: docker-push docker-push-hugo docker-push-support docker-push: docker-push-hugo docker-push-support diff --git a/README.md b/README.md index 516dc7bf..cf4f848f 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,6 @@ In order to run the Flux site locally, you need to install: Once those tools are installed, fetch the assets necessary to run the site: -```cli -npm install -python3 -m pip install -r requirements.txt -``` - Then run the site in "server" mode: ```cli @@ -45,13 +40,19 @@ make serve Navigate to to see the site running in your browser. As you make updates to the site, the browser will immediately update to reflect those changes. +> **Note**: Sometimes you need to clear the output of previous builds, e.g. when the structure of imported documentation has changed. If you see duplicate pages being served, run +> ``` +> git clean -fx -- content/ +> ``` +> and then run `make serve` again. The duplicates will then be gone. + ## Publishing the site The Flux website is published automatically by [Netlify](https://netlify.com) when changes are pushed to the `main` branch. The site does not need to be published manually. ### Preview builds -When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **deploy/netlify** link in the pull request window. +When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **Details** link of the **netlify/fluxcd/deploy-preview** check at the bottom of the pull request page. ## Local Development (docker) diff --git a/content/en/flux/components/helm/.gitignore b/content/en/flux/components/helm/.gitignore index d57ee8b5..5264c36c 100644 --- a/content/en/flux/components/helm/.gitignore +++ b/content/en/flux/components/helm/.gitignore @@ -1,2 +1,2 @@ -api.md +api/v*.md helmreleases.md diff --git a/content/en/flux/components/helm/api/_index.md b/content/en/flux/components/helm/api/_index.md new file mode 100644 index 00000000..ea8b9134 --- /dev/null +++ b/content/en/flux/components/helm/api/_index.md @@ -0,0 +1,5 @@ +--- +title: API Reference +linkTitle: API Reference +weight: 1000 +--- diff --git a/content/en/flux/components/helm/options.md b/content/en/flux/components/helm/options.md index d781c61d..71c4d7fb 100644 --- a/content/en/flux/components/helm/options.md +++ b/content/en/flux/components/helm/options.md @@ -2,7 +2,7 @@ title: Controller Options linkTitle: Controller Options description: "Controller command flags and defaults." -weight: 1000 +weight: 1 --- To customise the controller options at install time, diff --git a/content/en/flux/components/image/.gitignore b/content/en/flux/components/image/.gitignore index 84061d40..3af01be5 100644 --- a/content/en/flux/components/image/.gitignore +++ b/content/en/flux/components/image/.gitignore @@ -1,5 +1,5 @@ -automation-api.md +automation-api/v*.md +reflector-api/v*.md imagepolicies.md imagerepositories.md imageupdateautomations.md -reflector-api.md diff --git a/content/en/flux/components/image/automation-api/_index.md b/content/en/flux/components/image/automation-api/_index.md new file mode 100644 index 00000000..9a653328 --- /dev/null +++ b/content/en/flux/components/image/automation-api/_index.md @@ -0,0 +1,5 @@ +--- +title: Image Update Automation API Reference +linkTitle: Image Update Automation API Reference +weight: 70 +--- diff --git a/content/en/flux/components/image/options.md b/content/en/flux/components/image/options.md index 9862735b..710d6d9e 100644 --- a/content/en/flux/components/image/options.md +++ b/content/en/flux/components/image/options.md @@ -2,7 +2,7 @@ title: Controller Options linkTitle: Controller Options description: "Controller command flags and defaults." -weight: 1000 +weight: 1 --- To customise the controller options at install time, diff --git a/content/en/flux/components/image/reflector-api/_index.md b/content/en/flux/components/image/reflector-api/_index.md new file mode 100644 index 00000000..b9765032 --- /dev/null +++ b/content/en/flux/components/image/reflector-api/_index.md @@ -0,0 +1,5 @@ +--- +title: Image Reflector API Reference +linkTitle: Image Reflector API Reference +weight: 70 +--- diff --git a/content/en/flux/components/kustomize/.gitignore b/content/en/flux/components/kustomize/.gitignore index db062c04..52199d1c 100644 --- a/content/en/flux/components/kustomize/.gitignore +++ b/content/en/flux/components/kustomize/.gitignore @@ -1,2 +1,2 @@ api/v*.md -kustomization.md +kustomizations.md diff --git a/content/en/flux/components/notification/.gitignore b/content/en/flux/components/notification/.gitignore index 772e150e..d523d6fc 100644 --- a/content/en/flux/components/notification/.gitignore +++ b/content/en/flux/components/notification/.gitignore @@ -1,5 +1,5 @@ -alert.md +alerts.md api/v*.md -event.md -provider.md -receiver.md +events.md +providers.md +receivers.md diff --git a/hack/import-flux2-assets.sh b/hack/import-flux2-assets.sh index e937c9a3..5a2d8aef 100755 --- a/hack/import-flux2-assets.sh +++ b/hack/import-flux2-assets.sh @@ -1,9 +1,11 @@ -#!/bin/sh -x +#!/usr/bin/env bash + +set -euxo pipefail COMPONENTS_DIR="content/en/flux/components" FLUX_DIR="content/en/flux/cmd" -if [ -z "${GITHUB_USER}" ]; then +if [ -z "${GITHUB_USER:-}" ]; then GITHUB_USER=fluxcdbot fi @@ -19,7 +21,7 @@ fatal() { # Set os, fatal if operating system not supported setup_verify_os() { - if [ -z "${OS}" ]; then + if [ -z "${OS:-}" ]; then OS=$(uname) fi case ${OS} in @@ -36,7 +38,7 @@ setup_verify_os() { # Set arch, fatal if architecture not supported setup_verify_arch() { - if [ -z "${ARCH}" ]; then + if [ -z "${ARCH:-}" ]; then ARCH=$(uname -m) fi case ${ARCH} in @@ -58,22 +60,10 @@ setup_verify_arch() { } -controller_version() { - url=https://api.github.com/repos/fluxcd/$1/releases - out=$(curl -w "%{http_code}" -o /tmp/releases -u "$GITHUB_USER:$GITHUB_TOKEN" -s "$url") - if [ "${out}" -ne "200" ] ; then - echo "error fetching releases from $url: $out" - cat /tmp/releases - exit 1 - fi - - jq -r '.[] | .tag_name' < /tmp/releases | sed 's/.*\///' | awk '{ if ($1 ~ /-/) print; else print $0"_" ; }' | sort -rV | sed 's/_$//' | head -n1 -} - gen_crd_doc() { URL="$1" DEST="$2" - HUGETABLE="$3" + HUGETABLE="${3:-}" TMP="$(mktemp)" curl -u "$GITHUB_USER:$GITHUB_TOKEN" -# -Lf "$URL" > "$TMP" @@ -85,7 +75,7 @@ gen_crd_doc() { # 3. While others go # Here is the title you're looking for... # - FIRST_LINE="$(grep -vE "^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2)" + WEIGHT="$(grep -E '^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2 || true)" if [ -z "${WEIGHT}" ] ; then WEIGHT=0 fi @@ -119,55 +109,47 @@ gen_crd_doc() { fi } -{ - # source-controller CRDs - SOURCE_VER="$(controller_version source-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1/source.md" "$COMPONENTS_DIR/source/api/v1.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1beta2/source.md" "$COMPONENTS_DIR/source/api/v1beta2.md" "HUGETABLE" - SOURCE_CRD_VER="v1" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/gitrepositories.md" "$COMPONENTS_DIR/source/gitrepositories.md" - SOURCE_CRD_VER="v1beta2" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/ocirepositories.md" "$COMPONENTS_DIR/source/ocirepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmrepositories.md" "$COMPONENTS_DIR/source/helmrepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmcharts.md" "$COMPONENTS_DIR/source/helmcharts.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/buckets.md" "$COMPONENTS_DIR/source/buckets.md" +function all_versions { + for crd in $1 ; do + echo "${crd##*,}" + done | sort | uniq } -{ - # kustomize-controller CRDs - KUSTOMIZE_VER="$(controller_version kustomize-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/api/v1/kustomize.md" "$COMPONENTS_DIR/kustomize/api/v1.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/spec/v1/kustomization.md" "$COMPONENTS_DIR/kustomize/kustomization.md" -} +function gen_ctrl_docs { + flux_version=${1} + ctrl=${2} + ctrl_short=${ctrl%%-controller} + ctrl_out=${ctrl%%-*} # this is necessary to collect IAC and IRC together in the "image" folder -{ - # helm-controller CRDs - HELM_VER="$(controller_version helm-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/api/helmrelease.md" "$COMPONENTS_DIR/helm/api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/spec/v2beta1/helmreleases.md" "$COMPONENTS_DIR/helm/helmreleases.md" -} + api_out="api" + if [ "${ctrl_short}" = "image-reflector" ] ; then + api_out="reflector-api" + fi + if [ "${ctrl_short}" = "image-automation" ] ; then + api_out="automation-api" + fi -{ - # notification-controller CRDs - NOTIFICATION_VER="$(controller_version notification-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1/notification.md" "$COMPONENTS_DIR/notification/api/v1.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1beta2/notification.md" "$COMPONENTS_DIR/notification/api/v1beta2.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/notification/event.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/alerts.md" "$COMPONENTS_DIR/notification/alert.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/providers.md" "$COMPONENTS_DIR/notification/provider.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1/receivers.md" "$COMPONENTS_DIR/notification/receiver.md" -} + ks_url=$(curl -sL "https://raw.githubusercontent.com/fluxcd/flux2/${flux_version}/manifests/bases/${ctrl}/kustomization.yaml" | yq '.resources[]|select(. == "*crds.yaml*")') + ctrl_version=$(echo "${ks_url}" | cut -d/ -f8) -{ - # image-*-controller CRDs; these use the same API group - IMG_REFL_VER="$(controller_version image-reflector-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/api/image-reflector.md" "$COMPONENTS_DIR/image/reflector-api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagerepositories.md" "$COMPONENTS_DIR/image/imagerepositories.md" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagepolicies.md" "$COMPONENTS_DIR/image/imagepolicies.md" + crds=$(curl -sL "${ks_url}" | yq ea '[[.metadata.name, .spec.versions[] | select(.storage == "true").name]]' -o csv) + for api_version in $(all_versions "${crds}") ; do + doc_url=https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/api/${api_version}/${ctrl_short}.md + gen_crd_doc "${doc_url}" "$COMPONENTS_DIR/${ctrl_out}/${api_out}/${api_version}.md" "HUGETABLE" - IMG_AUTO_VER="$(controller_version image-automation-controller)" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/api/image-automation.md" "$COMPONENTS_DIR/image/automation-api.md" "HUGETABLE" - gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/spec/v1beta1/imageupdateautomations.md" "$COMPONENTS_DIR/image/imageupdateautomations.md" + done + + for crd in ${crds} ; do + name=${crd%%,*} + name=${name%%.*} + version=${crd##*,} + gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/${version}/${name}.md" "$COMPONENTS_DIR/${ctrl_out}/${name}.md" + done + + # special case: n-c's `Events` type is not a CRD but needs to be documented, too. + if [ "${ctrl}" = "notification-controller" ] ; then + gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/${ctrl_out}/events.md" + fi } { @@ -176,11 +158,31 @@ gen_crd_doc() { setup_verify_arch TMP="$(mktemp -d)" - TMP_METADATA="$TMP/flux.json" TMP_BIN="$TMP/flux.tar.gz" - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_METADATA}" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases/latest" - VERSION_FLUX=$(grep '"tag_name":' "${TMP_METADATA}" | sed -E 's/.*"([^"]+)".*/\1/' | cut -c 2-) + if [ -z "${BRANCH:-}" ] ; then + fatal "BRANCH environment variable not set" + fi + + if [[ "${BRANCH}" =~ ^pull/[[:digit:]]*/head$ ]] ; then + BRANCH=$(curl -sSfL "https://api.github.com/repos/fluxcd/website/pulls/$(echo ${BRANCH}|cut -d/ -f2)" | jq .base.ref -r) + fi + + VERSION_FLUX= + for tag in $(curl -u "$GITHUB_USER:$GITHUB_TOKEN" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases" | jq .[].tag_name -r) ; do + if [ "${BRANCH}" = "main" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + if [ "${tag%.*}" = "${BRANCH/-/.}" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + done + + if [ -z "${VERSION_FLUX}" ] ; then + fatal "No Flux version found matching branch '${BRANCH}'" + fi curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz" tar xfz "${TMP_BIN}" -C "${TMP}" @@ -191,11 +193,37 @@ gen_crd_doc() { rm -rf "$TMP" } +{ + # source-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "source-controller" +} + +{ + # kustomize-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "kustomize-controller" +} + +{ + # helm-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "helm-controller" +} + +{ + # notification-controller CRDs + gen_ctrl_docs "v${VERSION_FLUX}" "notification-controller" +} + +{ + # image-*-controller CRDs; these use the same API group + gen_ctrl_docs "v${VERSION_FLUX}" "image-reflector-controller" + gen_ctrl_docs "v${VERSION_FLUX}" "image-automation-controller" +} + { # provide Flux install script if [ ! -d static ]; then mkdir static fi - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf https://raw.githubusercontent.com/fluxcd/flux2/main/install/flux.sh -o static/install.sh + curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf "https://raw.githubusercontent.com/fluxcd/flux2/v${VERSION_FLUX}/install/flux.sh" -o static/install.sh }