Merge pull request #142 from fluxcd/update-kyverno

Update kyverno to v1.14.4

.github/actions/tools/Dockerfile

@@ -1,6 +0,0 @@
-FROM stefanprodan/alpine-base:latest
-
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-USER 1001
-ENTRYPOINT ["/entrypoint.sh"]

.github/actions/tools/action.yml

@@ -1,9 +0,0 @@
-name: 'kustomize'
-description: 'A GitHub Action with Kubernetes tools'
-author: 'Stefan Prodan'
-branding:
-  icon: 'command'
-  color: 'blue'
-runs:
-  using: 'docker'
-  image: 'Dockerfile'

.github/actions/tools/entrypoint.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-YQ_VERSION="v4.6.1"
-KUSTOMIZE_VERSION="4.1.3"
-KUBEVAL_VERSION="v0.16.1"
-
-mkdir -p $GITHUB_WORKSPACE/bin
-cd $GITHUB_WORKSPACE/bin
-
-curl -sL https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o yq
-
-chmod +x $GITHUB_WORKSPACE/bin/yq
-
-kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \
-curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | \
-tar xz
-
-chmod +x $GITHUB_WORKSPACE/bin/kustomize
-
-curl -sL https://github.com/instrumenta/kubeval/releases/download/${KUBEVAL_VERSION}/kubeval-linux-amd64.tar.gz | \
-tar xz
-
-chmod +x $GITHUB_WORKSPACE/bin/kubeval
-
-echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH
-echo "$RUNNER_WORKSPACE/$(basename $GITHUB_REPOSITORY)/bin" >> $GITHUB_PATH

.github/workflows/e2e.yaml

@@ -1,6 +1,7 @@
 name: e2e
 
 on:
+  workflow_dispatch:
   push:
     branches: [ '*' ]
 
@@ -9,16 +10,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-      - name: Setup tools
-        uses: ./.github/actions/tools
       - name: Setup Flux
         uses: fluxcd/flux2/action@main
       - name: Setup Kubernetes
-        uses: engineerd/setup-kind@v0.5.0
+        uses: helm/kind-action@v1.7.0
         with:
-          version: v0.11.1
+          cluster_name: flux
-          image: kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047
       - name: Install Flux in Kubernetes Kind
         run: flux install --log-level debug
       - name: Setup cluster reconciliation
@@ -26,31 +24,35 @@ jobs:
           flux create source git flux-system \
           --interval=15m \
           --url=${{ github.event.repository.html_url }} \
-          --branch=${GITHUB_REF#refs/heads/}
+          --branch=${GITHUB_REF#refs/heads/} \
+          --ignore-paths="./clusters/**/flux-system/"
+
           flux create kustomization flux-system \
           --interval=15m \
           --source=flux-system \
           --path=./clusters/staging
       - name: Verify cluster reconciliation
         run: |
-          kubectl -n flux-system wait kustomization/kyverno --for=condition=ready --timeout=1m
+          kubectl -n flux-system wait kustomization/kyverno --for=condition=ready --timeout=5m
-          kubectl -n flux-system wait kustomization/kyverno-policies --for=condition=ready --timeout=1m
+          kubectl -n flux-system wait kustomization/kyverno-controller --for=condition=ready --timeout=3m
+          kubectl -n flux-system wait kustomization/kyverno-policies --for=condition=ready --timeout=3m
           kubectl -n flux-system wait kustomization/tenants --for=condition=ready --timeout=3m
       - name: Verify tenant reconciliation
         run: |
           kubectl -n apps wait kustomization/dev-team --for=condition=ready --timeout=1m
           kubectl -n apps wait helmrelease/podinfo --for=condition=ready --timeout=1m
-      - name: Debug failure
+      - name: List reconciliations
-        if: failure()
+        run: |
+          flux get all --all-namespaces
+      - name: Logs
+        if: always()
         run: |
           kubectl -n flux-system get all
           kubectl -n flux-system logs deploy/source-controller
           kubectl -n flux-system logs deploy/kustomize-controller
-          kubectl -n flux-system logs deploy/helm-controller
-          kubectl -n flux-system logs deploy/notification-controller
           kubectl -n kyverno get all
-          kubectl -n kyverno logs deploy/kyverno
+          kubectl -n kyverno describe po 
-          kubectl get clusterpolicy flux-multi-tenancy -oyaml
+          kubectl -n kyverno logs -l app.kubernetes.io/part-of=kyverno
+          kubectl get clusterpolicies.kyverno.io -oyaml
           flux get sources all --all-namespaces
           flux get ks --all-namespaces
-

.github/workflows/test.yaml

@@ -11,8 +11,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-      - name: Setup tools
+      - name: Setup yq
-        uses: ./.github/actions/tools
+        uses: fluxcd/pkg/actions/yq@main
+      - name: Setup kubeconform
+        uses: fluxcd/pkg/actions/kubeconform@main
+      - name: Setup kustomize
+        uses: fluxcd/pkg/actions/kustomize@main
       - name: Validate manifests
         run: ./scripts/validate.sh

.github/workflows/update.yaml

@@ -10,9 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-      - name: Setup tools
+      - name: Setup yq
-        uses: ./.github/actions/tools
+        uses: fluxcd/pkg/actions/yq@main
       - name: Fetch latest version
         id: version
         run: |
@@ -25,9 +25,9 @@ jobs:
           head -n1)
           echo ::set-output name=NUMBER::${VERSION}
       - name: Patch version
-        run: |
+        env:
-          export URL="https://raw.githubusercontent.com/kyverno/kyverno/${{ steps.version.outputs.NUMBER }}/definitions/release/install.yaml"
+          KYVERNO_VERSION: ${{ steps.version.outputs.NUMBER }}
-          yq e '.resources[0]=strenv(URL)' -i ./infrastructure/kyverno/kustomization.yaml
+        run: yq e '.spec.ref.tag=strenv(KYVERNO_VERSION)' -i ./infrastructure/kyverno/source.yaml
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:

.sourceignore

@@ -1,5 +1,5 @@
 # Flux ignore
-# https://toolkit.fluxcd.io/components/source/gitrepositories/#excluding-files
+# https://fluxcd.io/flux/components/source/gitrepositories/#excluding-files
 docs/
 scripts/
 *.md

README.md

@@ -65,7 +65,83 @@ A [tenant repository](https://github.com/fluxcd/flux2-multi-tenancy/tree/dev-tea
     └── podinfo-values.yaml
 ```
 
-## Onboard tenants
+## Bootstrap the staging cluster
+
+Install the Flux CLI and fork this repository on your personal GitHub account
+and export your GitHub username and repo name:
+
+```sh
+export GITHUB_USER=<your-username>
+export GITHUB_REPO=<repository-name>
+```
+
+Verify that your staging cluster satisfies the prerequisites with:
+
+```sh
+flux check --pre
+```
+
+Set the `--context` argument to the kubectl context to your staging cluster and bootstrap Flux:
+
+```sh
+flux bootstrap github \
+    --context=your-staging-context \
+    --owner=${GITHUB_USER} \
+    --repository=${GITHUB_REPO} \
+    --branch=main \
+    --personal \
+    --path=clusters/staging
+```
+
+At this point flux cli will ask you for your `GITHUB_TOKEN` (a.k.a [Personal Access Token]).
+
+> **NOTE:** The `GITHUB_TOKEN` is used exclusively by the flux CLI during the bootstrapping process,
+> and does not leave your machine. The credential is used for
+> configuring the GitHub repository and registering the deploy key.
+
+The bootstrap command commits the manifests for the Flux components in `clusters/staging/flux-system` dir
+and creates a deploy key with read-only access on GitHub, so it can pull changes inside the cluster.
+
+Wait for the staging cluster reconciliation to finish:
+
+```console
+$ flux get kustomizations --watch
+NAME            	READY  	MESSAGE                                                        	
+flux-system     	True   	Applied revision: main/616001c38e7bc81b00ef2c65ac8cfd58140155b8	
+kyverno         	Unknown	Reconciliation in progress
+kyverno-policies	False  	Dependency 'flux-system/kyverno' is not ready
+tenants         	False  	Dependency 'flux-system/kyverno-policies' is not ready
+```
+
+Verify that the tenant Git repository has been cloned:
+
+```console
+$ flux -n apps get sources git
+NAME    	READY	MESSAGE 
+dev-team	True 	Fetched revision: dev-team/ca8ec25405cc03f2f374d2f35f9299d84ced01e4
+```
+
+Verify that the tenant Helm repository index has been downloaded:
+
+```console
+$ flux -n apps get sources helm
+NAME   	READY	MESSAGE
+podinfo	True 	Fetched revision: 2022-05-23T10:09:58.648748663Z
+```
+
+Wait for the demo app to be installed:
+
+```console
+$ watch flux -n apps get helmreleases
+NAME   	READY	MESSAGE                         	REVISION	SUSPENDED 
+podinfo	True 	Release reconciliation succeeded	5.0.3   	False 
+```
+
+To expand on this example, check the [enforce tenant isolation](#enforce-tenant-isolation) for security related considerations. 
+
+[Personal Access Token]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
+
+### Onboard new tenants
 
 The Flux CLI offers commands to generate the Kubernetes manifests needed to define tenants.
 
@@ -104,14 +180,14 @@ flux create kustomization dev-team \
 Create the base `kustomization.yaml` file:
 
 ```sh
-cd ./tenants/base/dev-team/ && kustomize create --autodetect
+cd ./tenants/base/dev-team/ && kustomize create --autodetect --namespace apps 
 ```
 
 Create the staging overlay and set the path to the staging dir inside the tenant repository:
 
 ```sh
 cat << EOF | tee ./tenants/staging/dev-team-patch.yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: dev-team
@@ -123,10 +199,11 @@ EOF
 cat << EOF | tee ./tenants/staging/kustomization.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: apps
 resources:
   - ../base/dev-team
-patchesStrategicMerge:
+patches:
-  - dev-team-patch.yaml
+  - path: dev-team-patch.yaml
 EOF
 ```
 
@@ -135,13 +212,182 @@ dev-team's repository, and it will reconcile the `./staging` directory from the
 using the `dev-team` service account. Since that service account is restricted to the `apps` namespace,
 the dev-team repository must contain Kubernetes objects scoped to the `apps` namespace only.
 
+#### Tenant onboarding via Kyverno
+
+Alternatively to the `flux create tenant` approach, Kyverno's [resource generation] feature can
+be leveraged to the same effect.
+
+[resource generation]: https://kyverno.io/docs/writing-policies/generate/
+
 ## Enforce tenant isolation
 
 To enforce tenant isolation, cluster admins must configure Flux to reconcile 
 the `Kustomization` and `HelmRelease` kinds by impersonating a service account
-from the namespace where these objects are created. In order to make the
+from the namespace where these objects are created. 
-`spec.ServiceAccountName` field mandatory, you should use a validation webhook, for example
+
-[Kyverno](https://github.com/kyverno/kyverno) or [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper).
+Flux has built-in [multi-tenancy lockdown] features which enables tenant isolation 
+at Control Plane level without the need of external admission controllers (e.g. Kyverno). The
+recommended patch:
+
+- Enforce controllers to block cross namespace references.
+  Meaning that a tenant can’t use another tenant’s sources or subscribe to their events.
+- Deny accesses to Kustomize remote bases, thus ensuring all resources refer to local files. 
+  Meaning that only approved Flux Sources can affect the cluster-state.
+- Sets a default service account via `--default-service-account` to `kustomize-controller` and `helm-controller`.
+  Meaning that, if a tenant does not specify a service account in a Flux `Kustomization` or 
+  `HelmRelease`, it would automatically default to said account. 
+
+> **NOTE:** It is recommended that the default service account has no privileges.
+> And each named service account used observes the least privilege model.
+
+This repository applies this patch automatically via
+[kustomization.yaml](clusters/production/flux-system/kustomization.yaml) in both clusters.
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gotk-components.yaml
+  - gotk-sync.yaml
+patches:
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-cross-namespace-refs=true
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller|notification-controller|image-reflector-controller|image-automation-controller)"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-remote-bases=true
+    target:
+      kind: Deployment
+      name: "kustomize-controller"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --default-service-account=default
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller)"
+  - patch: |
+      - op: add
+        path: /spec/serviceAccountName
+        value: kustomize-controller
+    target:
+      kind: Kustomization
+      name: "flux-system"
+```
+
+### Side Effects
+
+When Flux is bootstrapped with the patch both `kustomize-controller` and `helm-controller` will impersonate the `default`
+service account in the tenant namespace when applying changes to the cluster. The `default` service account 
+exist in all namespaces and should always be kept without any privileges.
+
+To enable a tenant to operate, a service account must be created with the required permissions and its name set 
+to the `spec.serviceAccountName` of all `Kustomization` and `HelmRelease` resources the tenant has.
+
+### Tenancy policies
+
+Depending on the aimed security posture, the Platform Admin may impose additional policies to enforce specific 
+behaviours. Below are a few consideration points, some of which are already implemented in this repository.
+
+#### Image provenance
+
+Assuring the provenance of container images across a cluster can be achieved on several ways.
+
+The [verify-flux-images policy](infrastructure/kyverno-policies/verify-flux-images.yaml)
+ensures that all Flux images used are the ones built and signed by the Flux team:
+
+```yaml
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: verify-flux-images
+spec:
+  validationFailureAction: enforce
+  background: false
+  webhookTimeoutSeconds: 30
+  failurePolicy: Fail
+  rules:
+    - name: verify-cosign-signature
+      match:
+        resources:
+          kinds:
+            - Pod
+      verifyImages:
+        - imageReferences:
+            - "ghcr.io/fluxcd/source-controller:*"
+            - "ghcr.io/fluxcd/kustomize-controller:*"
+            - "ghcr.io/fluxcd/helm-controller:*"
+            - "ghcr.io/fluxcd/notification-controller:*"
+          attestors:
+            - entries:
+                - keyless:
+                    subject: "https://github.com/fluxcd/*"
+                    issuer: "https://token.actions.githubusercontent.com"
+                    rekor:
+                      url: https://rekor.sigstore.dev
+```
+
+Other policies to explore:
+- Restrict what repositories can be accessed in each cluster. Some deployments may need this to be environment-specific.
+- Align image policies with pods that require `securityContext` that are highly privileged.
+
+#### Flux Sources
+
+Flux uses sources to define the origin of flux manifests. Some deployments may require that 
+all of them come from a specific GitHub Organisation, as the
+[verify-git-repositories policy](infrastructure/kyverno-policies/verify-git-repositories.yaml) shows:
+
+```yaml
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: verify-git-repositories
+spec:
+  validationFailureAction: audit # Change to 'enforce' once the specific org url is set.
+  rules:
+    - name: github-repositories-only
+      exclude:
+        resources:
+          namespaces:
+            - flux-system
+      match:
+        resources:
+          kinds:
+            - GitRepository
+      validate:
+        message: ".spec.url must be from a repository within the organisation X"
+        anyPattern:
+        - spec:
+            url: "https://github.com/fluxcd/?*" # repositories in fluxcd via https
+        - spec:
+            url: "ssh://git@github.com:fluxcd/?*" # repositories in fluxcd via ssh
+```
+
+Other policies to explore:
+- Expand the policies to `HelmRepository` and `Bucket`.
+- For `HelmRepository` and `GitRepository` consider which protocols should be allowed.
+- For `Bucket`, consider restrictions on providers and regions.
+
+#### Make serviceAccountName mandatory
+
+The lockdown patch sets a default service account that is applied to any `Kustomization` and `HelmRelease` 
+instances that have no `spec.ServiceAccountName` set.
+
+If the recommended best practices above are followed, such instances won't be able to apply changes to
+a cluster as the default service account has no permissions to do so. 
+
+An additional extra could be taken to make the `spec.ServiceAccountName` field  mandatory via a validation 
+webhook, for example [Kyverno](https://github.com/kyverno/kyverno) or
+[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper).
+Resulting on `Kustomization` and `HelmRelease` instances not being admitted when `spec.ServiceAccountName` is not set.
+
+#### Reconciliation hierarchy
+
 On cluster bootstrap, you need to configure Flux to deploy the validation webhook and its policies before 
 reconciling the tenants repositories.
 
@@ -162,7 +408,7 @@ First we setup the reconciliation of custom resource definitions and their contr
 example we'll use [Kyverno](https://github.com/kyverno/kyverno):
 
 ```yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno
@@ -178,11 +424,11 @@ spec:
   timeout: 5m
 ```
 
-Then we setup [cluster policies](./infrastructure/kyverno-policies/flux-multi-tenancy.yaml) 
+Then we setup [cluster policies](./infrastructure/kyverno-policies/verify-git-repositories.yaml) 
-(Kyverno custom resources) to enforce tenant isolation:
+(Kyverno custom resources) to enforce a specific GitHub Organisation:
 
 ```yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno-policies
@@ -203,7 +449,7 @@ With `dependsOn` we tell Flux to install Kyverno before deploying the cluster po
 And finally we setup the reconciliation for the tenants workloads with:
 
 ```yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: tenants
@@ -219,75 +465,8 @@ spec:
   prune: true
 ```
 
-With the above configuration, we ensure that the Kyverno validation webhook will reject `Kustomizations` and 
+With the above configuration, we ensure that the Kyverno validation webhook will reject `GitRepository`
-`HelmReleases` that don't specify a service account name when deployed in a tenant's namespace.
+that don't originate from a specific GitHub Organisation, in our case `fluxcd`.
-
-## Bootstrap the staging cluster
-
-Install the Flux CLI and fork this repository on your personal GitHub account
-and export your GitHub access token, username and repo name:
-
-```sh
-export GITHUB_TOKEN=<your-token>
-export GITHUB_USER=<your-username>
-export GITHUB_REPO=<repository-name>
-```
-
-Verify that your staging cluster satisfies the prerequisites with:
-
-```sh
-flux check --pre
-```
-
-Set the `--context` argument to the kubectl context to your staging cluster and bootstrap Flux:
-
-```sh
-flux bootstrap github \
-    --context=your-staging-context \
-    --owner=${GITHUB_USER} \
-    --repository=${GITHUB_REPO} \
-    --branch=main \
-    --personal \
-    --path=clusters/staging
-```
-
-The bootstrap command commits the manifests for the Flux components in `clusters/staging/flux-system` dir
-and creates a deploy key with read-only access on GitHub, so it can pull changes inside the cluster.
-
-Wait for the staging cluster reconciliation to finish:
-
-```console
-$ watch flux get kustomization
-NAME            	READY  	MESSAGE                                                        	
-flux-system     	True   	Applied revision: main/616001c38e7bc81b00ef2c65ac8cfd58140155b8	
-kyverno         	Unknown	Reconciliation in progress
-kyverno-policies	False  	Dependency 'flux-system/kyverno' is not ready
-tenants         	False  	Dependency 'flux-system/kyverno-policies' is not ready
-```
-
-Verify that the tenant Git repository has been cloned:
-
-```console
-$ flux -n apps get sources git
-NAME    	READY	MESSAGE 
-dev-team	True 	Fetched revision: dev-team/ca8ec25405cc03f2f374d2f35f9299d84ced01e4
-```
-
-Verify that the tenant Helm repository index has been downloaded:
-
-```console
-$ flux -n apps get sources helm
-NAME   	READY	MESSAGE
-podinfo	True 	Fetched revision: 2020-10-28T10:09:58.648748663Z
-```
-
-Wait for the demo app to be installed:
-
-```console
-$ watch flux -n apps get helmreleases
-NAME   	READY	MESSAGE                         	REVISION	SUSPENDED 
-podinfo	True 	Release reconciliation succeeded	5.0.3   	False 
-```
 
 ## Onboard tenants with private repositories
 
@@ -298,7 +477,7 @@ in the platform admin repository as a Kubernetes secret.
 ### Encrypt Kubernetes secrets in Git
 
 In order to store credentials safely in a Git repository, you can use Mozilla's
-SOPS CLI to encrypt Kubernetes secrets with OpenPGP or KMS.
+SOPS CLI to encrypt Kubernetes secrets with OpenPGP, Age or KMS.
 
 Install [gnupg](https://www.gnupg.org/) and [sops](https://github.com/mozilla/sops):
 
@@ -345,7 +524,7 @@ flux -n apps create secret git dev-team-auth \
 Print the SSH public key and add it as a read-only deploy key to the dev-team repository:
 
 ```sh
-yq read git-auth.yaml 'data."identity.pub"' | base64 --decode
+yq eval '.stringData."identity.pub"' ./tenants/base/dev-team/auth.yaml
 ```
 
 ### Git over HTTP/S
@@ -427,5 +606,10 @@ a pull request is merged into the main branch and synced on the cluster.
 
 This repository contains the following GitHub CI workflows:
 
-* the [test](./.github/workflows/test.yaml) workflow validates the Kubernetes manifests and Kustomize overlays with kubeval
+* the [test](./.github/workflows/test.yaml) workflow validates the Kubernetes manifests
-* the [e2e](./.github/workflows/e2e.yaml) workflow starts a Kubernetes cluster in CI and tests the staging setup by running Flux in Kubernetes Kind
+  and Kustomize overlays with [kubeconform](https://github.com/yannh/kubeconform)
+* the [e2e](./.github/workflows/e2e.yaml) workflow starts a Kubernetes cluster in CI
+  and tests the staging setup by running Flux in Kubernetes Kind
+
+
+[multi-tenancy lockdown]: https://fluxcd.io/flux/installation/configuration/multitenancy/

clusters/production/flux-system/gotk-components.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/production/flux-system/gotk-sync.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/production/flux-system/kustomization.yaml

@@ -0,0 +1,34 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gotk-components.yaml
+  - gotk-sync.yaml
+patches:
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-cross-namespace-refs=true
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller|notification-controller|image-reflector-controller|image-automation-controller)"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-remote-bases=true
+    target:
+      kind: Deployment
+      name: "kustomize-controller"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --default-service-account=default
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller)"
+  - patch: |
+      - op: add
+        path: /spec/serviceAccountName
+        value: kustomize-controller
+    target:
+      kind: Kustomization
+      name: "flux-system"

clusters/production/infrastructure.yaml

@@ -1,4 +1,4 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno
@@ -8,12 +8,13 @@ spec:
   sourceRef:
     kind: GitRepository
     name: flux-system
+  serviceAccountName: kustomize-controller
   path: ./infrastructure/kyverno
   prune: true
   wait: true
   timeout: 5m
 ---
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno-policies
@@ -25,5 +26,6 @@ spec:
   sourceRef:
     kind: GitRepository
     name: flux-system
+  serviceAccountName: kustomize-controller
   path: ./infrastructure/kyverno-policies
   prune: true

clusters/production/tenants.yaml

@@ -1,4 +1,4 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: tenants
@@ -7,6 +7,7 @@ spec:
   dependsOn:
     - name: kyverno-policies
   interval: 5m
+  serviceAccountName: kustomize-controller
   sourceRef:
     kind: GitRepository
     name: flux-system

clusters/staging/flux-system/gotk-components.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/staging/flux-system/gotk-sync.yaml

@@ -0,0 +1 @@
+# This file will be generated automatically by flux boostrap.

clusters/staging/flux-system/kustomization.yaml

@@ -0,0 +1,34 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gotk-components.yaml
+  - gotk-sync.yaml
+patches:
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-cross-namespace-refs=true
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller|notification-controller|image-reflector-controller|image-automation-controller)"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --no-remote-bases=true
+    target:
+      kind: Deployment
+      name: "kustomize-controller"
+  - patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --default-service-account=default
+    target:
+      kind: Deployment
+      name: "(kustomize-controller|helm-controller)"
+  - patch: |
+      - op: add
+        path: /spec/serviceAccountName
+        value: kustomize-controller
+    target:
+      kind: Kustomization
+      name: "flux-system"

clusters/staging/infrastructure.yaml

@@ -1,19 +1,20 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno
   namespace: flux-system
 spec:
-  interval: 720m
+  interval: 720m0s
   sourceRef:
     kind: GitRepository
     name: flux-system
+  serviceAccountName: kustomize-controller
   path: ./infrastructure/kyverno
   prune: true
   wait: true
-  timeout: 5m
+  timeout: 10m
 ---
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: kyverno-policies
@@ -21,9 +22,10 @@ metadata:
 spec:
   dependsOn:
     - name: kyverno
-  interval: 15m
+  interval: 15m0s
   sourceRef:
     kind: GitRepository
     name: flux-system
+  serviceAccountName: kustomize-controller
   path: ./infrastructure/kyverno-policies
   prune: true

clusters/staging/tenants.yaml

@@ -1,4 +1,4 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: tenants
@@ -7,6 +7,7 @@ spec:
   dependsOn:
     - name: kyverno-policies
   interval: 5m
+  serviceAccountName: kustomize-controller
   sourceRef:
     kind: GitRepository
     name: flux-system

infrastructure/kyverno-policies/flux-multi-tenancy.yaml

@@ -1,64 +0,0 @@
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: flux-multi-tenancy
-spec:
-  validationFailureAction: enforce
-  rules:
-    - name: serviceAccountName
-      exclude:
-        resources:
-          namespaces:
-            - flux-system
-      match:
-        resources:
-          kinds:
-            - Kustomization
-            - HelmRelease
-      validate:
-        message: ".spec.serviceAccountName is required"
-        pattern:
-          spec:
-            serviceAccountName: "?*"
-    - name: kustomizationSourceRefNamespace
-      exclude:
-        resources:
-          namespaces:
-            - flux-system
-      match:
-        resources:
-          kinds:
-            - Kustomization
-      preconditions:
-        any:
-          - key: "{{request.object.spec.sourceRef.namespace}}"
-            operator: NotEquals
-            value: ""
-      validate:
-        message: "spec.sourceRef.namespace must be the same as metadata.namespace"
-        deny:
-          conditions:
-            - key: "{{request.object.spec.sourceRef.namespace}}"
-              operator: NotEquals
-              value:  "{{request.object.metadata.namespace}}"
-    - name: helmReleaseSourceRefNamespace
-      exclude:
-        resources:
-          namespaces:
-            - flux-system
-      match:
-        resources:
-          kinds:
-            - HelmRelease
-      preconditions:
-        any:
-          - key: "{{request.object.spec.chart.spec.sourceRef.namespace}}"
-            operator: NotEquals
-            value: ""
-      validate:
-        message: "spec.chart.spec.sourceRef.namespace must be the same as metadata.namespace"
-        deny:
-          conditions:
-            - key: "{{request.object.spec.chart.spec.sourceRef.namespace}}"
-              operator: NotEquals
-              value:  "{{request.object.metadata.namespace}}"

infrastructure/kyverno-policies/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - flux-multi-tenancy.yaml
+  - verify-flux-images.yaml
+  - verify-git-repositories.yaml

infrastructure/kyverno-policies/verify-flux-images.yaml

@@ -0,0 +1,39 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: verify-flux-images
+spec:
+  validationFailureAction: Audit
+  background: false
+  webhookTimeoutSeconds: 30
+  failurePolicy: Fail
+  rules:
+    - name: verify-cosign-signature
+      match:
+        any:
+        - resources:
+            kinds:
+              - Pod
+      verifyImages:
+        - imageReferences:
+            - "ghcr.io/fluxcd/source-controller:*"
+            - "ghcr.io/fluxcd/kustomize-controller:*"
+            - "ghcr.io/fluxcd/helm-controller:*"
+            - "ghcr.io/fluxcd/notification-controller:*"
+            - "ghcr.io/fluxcd/image-reflector-controller:*"
+            - "ghcr.io/fluxcd/image-automation-controller:*"
+            - "docker.io/fluxcd/source-controller:*"
+            - "docker.io/fluxcd/kustomize-controller:*"
+            - "docker.io/fluxcd/helm-controller:*"
+            - "docker.io/fluxcd/notification-controller:*"
+            - "docker.io/fluxcd/image-reflector-controller:*"
+            - "docker.io/fluxcd/image-automation-controller:*"
+          mutateDigest: false
+          verifyDigest: false
+          attestors:
+            - entries:
+                - keyless:
+                    subject: "https://github.com/fluxcd/*"
+                    issuer: "https://token.actions.githubusercontent.com"
+                    rekor:
+                      url: https://rekor.sigstore.dev

infrastructure/kyverno-policies/verify-git-repositories.yaml

@@ -0,0 +1,26 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: verify-git-repositories
+spec:  
+  # This provides users a working example of how an admin
+  # would be able to enforce git repository sources across
+  # all tenants.
+  validationFailureAction: Audit # Change to 'Enforce' once the specific org url is set.
+  rules:
+    - name: github-repositories-only
+      match:
+        any:
+        - resources:
+            kinds:
+              - GitRepository
+      exclude:
+        any:
+        - resources:
+            namespaces:
+              - flux-system
+      validate:
+        message: ".spec.url must be from a repository within the organisation X"
+        pattern:
+          spec:
+            url: https://github.com/fluxcd/?* | ssh://git@github.com:fluxcd/?*

infrastructure/kyverno/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - https://raw.githubusercontent.com/kyverno/kyverno/v1.5.1/definitions/release/install.yaml
+  - source.yaml
+  - sync.yaml

infrastructure/kyverno/source.yaml

@@ -0,0 +1,13 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: kyverno-controller
+  namespace: flux-system
+spec:
+  interval: 120m0s
+  provider: generic
+  url: oci://ghcr.io/kyverno/manifests/kyverno
+  ref:
+    tag: "v1.14.4"
+  verify:
+    provider: cosign

infrastructure/kyverno/sync.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kyverno-controller
+  namespace: flux-system
+spec:
+  interval: 720m0s
+  sourceRef:
+    kind: OCIRepository
+    name: kyverno-controller
+  serviceAccountName: kustomize-controller
+  path: ./
+  prune: true
+  wait: true
+  timeout: 5m

scripts/validate.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
 # This script downloads the Flux OpenAPI schemas, then it validates the
-# Flux custom resources and the kustomize overlays using kubeval.
+# Flux custom resources and the kustomize overlays using kubeconform.
 # This script is meant to be run locally and in CI before the changes
 # are merged on the main branch that's synced by Flux.
 
-# Copyright 2020 The Flux authors. All rights reserved.
+# Copyright 2023 The Flux authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,16 +19,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script is meant to be run locally and in CI to validate the Kubernetes
-# manifests (including Flux custom resources) before changes are merged into
-# the branch synced by Flux in-cluster.
-
 # Prerequisites
-# - yq v4.6
+# - yq v4.34
-# - kustomize v4.1
+# - kustomize v5.0
-# - kubeval v0.15
+# - kubeconform v0.6
 
 set -o errexit
+set -o pipefail
+
+# mirror kustomize-controller build options
+kustomize_flags=("--load-restrictor=LoadRestrictionsNone")
+kustomize_config="kustomization.yaml"
+
+# skip Kubernetes Secrets due to SOPS fields failing validation
+kubeconform_flags=("-skip=Secret")
+kubeconform_config=("-strict" "-ignore-missing-schemas" "-schema-location" "default" "-schema-location" "/tmp/flux-crd-schemas" "-verbose")
 
 echo "INFO - Downloading Flux OpenAPI schemas"
 mkdir -p /tmp/flux-crd-schemas/master-standalone-strict
@@ -43,22 +48,18 @@ done
 echo "INFO - Validating clusters"
 find ./clusters -maxdepth 2 -type f -name '*.yaml' -print0 | while IFS= read -r -d $'\0' file;
   do
-    kubeval ${file} --strict --ignore-missing-schemas --additional-schema-locations=file:///tmp/flux-crd-schemas
+    kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}" "${file}"
     if [[ ${PIPESTATUS[0]} != 0 ]]; then
       exit 1
     fi
 done
 
-# mirror kustomize-controller build options
-kustomize_flags="--load-restrictor=LoadRestrictionsNone --reorder=legacy"
-kustomize_config="kustomization.yaml"
-
 echo "INFO - Validating kustomize overlays"
 find . -type f -name $kustomize_config -print0 | while IFS= read -r -d $'\0' file;
   do
     echo "INFO - Validating kustomization ${file/%$kustomize_config}"
-    kustomize build "${file/%$kustomize_config}" $kustomize_flags | \
+    kustomize build "${file/%$kustomize_config}" "${kustomize_flags[@]}" | \
-      kubeval --ignore-missing-schemas --strict --additional-schema-locations=file:///tmp/flux-crd-schemas
+      kubeconform "${kubeconform_flags[@]}" "${kubeconform_config[@]}"
     if [[ ${PIPESTATUS[0]} != 0 ]]; then
       exit 1
     fi

tenants/base/dev-team/kustomization.yaml

@@ -1,5 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: apps
 resources:
   - rbac.yaml
   - sync.yaml

tenants/base/dev-team/sync.yaml

@@ -1,4 +1,4 @@
-apiVersion: source.toolkit.fluxcd.io/v1beta1
+apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
 metadata:
   name: dev-team
@@ -9,7 +9,7 @@ spec:
   ref:
     branch: dev-team
 ---
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: dev-team
@@ -21,6 +21,5 @@ spec:
     kind: GitRepository
     name: dev-team
   prune: true
-  validation: client
 
 

tenants/production/dev-team-patch.yaml

@@ -1,4 +1,4 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: dev-team

tenants/production/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: apps
 resources:
   - ../base/dev-team
-patchesStrategicMerge:
+patches:
-  - dev-team-patch.yaml
+  - path: dev-team-patch.yaml

tenants/staging/dev-team-patch.yaml

@@ -1,4 +1,4 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: dev-team

tenants/staging/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: apps
 resources:
   - ../base/dev-team
-patchesStrategicMerge:
+patches:
-  - dev-team-patch.yaml
+  - path: dev-team-patch.yaml