From b4b85029fb028d980a69c2e32dd2e834813a769c Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 Apr 2020 11:51:03 +0300 Subject: [PATCH 01/21] Add source API spec proposal --- docs/spec/README.md | 52 ++++++ docs/spec/v1alpha1/README.md | 10 ++ docs/spec/v1alpha1/gitrepositories.md | 212 +++++++++++++++++++++++++ docs/spec/v1alpha1/helmrepositories.md | 15 ++ 4 files changed, 289 insertions(+) create mode 100644 docs/spec/README.md create mode 100644 docs/spec/v1alpha1/README.md create mode 100644 docs/spec/v1alpha1/gitrepositories.md create mode 100644 docs/spec/v1alpha1/helmrepositories.md diff --git a/docs/spec/README.md b/docs/spec/README.md new file mode 100644 index 00000000..4e43ca52 --- /dev/null +++ b/docs/spec/README.md @@ -0,0 +1,52 @@ +# Source Controller Proposal + +## Context + +The desired state of a cluster is made out of Kubernetes objects, these objects are expressed in `.yaml` format and +are applied on the cluster by operators running inside the cluster. An operator's role is to fetch the Kubernetes +objects, run transformations on them and reconcile the cluster state with the resulting manifest. + +For an operator to acquire the resources that make up the desired state it needs to understand the communication +protocol and the authentication scheme, verify the authenticity of a source and deal with rate limits and retries. +In the FluxCD organization there are currently two operators that perform such operations. Both Flux and +Helm Operator connect to Git repositories to fetch Kubernetes objects, they need to maintain an up-to-date mirror +of one or several repos. Besides Git, Helm Operator needs to connect to Helm repositories hosted on public or +private HTTPS servers. + +## Motivation + +Each Flux or Helm Operator instance maintains its own Git repository mirror even if all of them +point to the same source. If the Git repository host becomes unavailable, the cluster state will diverge from the last +know desired state since the operators will stop the reconciliation due to pull errors. + +Decoupling the Kubernetes objects acquisition from the reconciliation process with an in-cluster +source manager would make Flux and Helm Operator resilient to outbound connectivity issues and would +simplify the state machine(s) that these controllers operate. + +Managing the source operations in a dedicated controller could allow Flux to compose the desire state of a cluster +from multiple source. +Further more the manifests transformation process could be performed by 3rd party tools +(e.g. kustomize, jk, tanka, cue run by Tekton pipelines or Kubernetes Jobs) +that would output the final state of the Kubernetes objects into an artifacts repository maintained by the source controller. + +## Goals + +The main goal is to define a set of Kubernetes objects that cluster admins and various automated operators +can interact with to offload the sources (e.g. Git and Helm repositories) +registration, authentication and resource fetching to a dedicated controller. + +The controller implementation will watch for source objects in a cluster and act on them. +The actions performed by the source controller could be: +* validate source definitions +* authenticate to sources and validate authenticity +* detect source changes based on update policies (semver) +* fetch resources on-demand and on-a-schedule +* package the fetched resources into a well known format (tar.gz) +* store the artifacts locally +* make the artifacts addressable by their source identifier (sha, version, ts) +* make the artifacts available in-cluster to interested 3rd parties +* notify interested 3rd parties of source changes and availability (status conditions, events, hooks) + +## API Specification + +* [v1alpha1](v1alpha1/README.md) diff --git a/docs/spec/v1alpha1/README.md b/docs/spec/v1alpha1/README.md new file mode 100644 index 00000000..23491974 --- /dev/null +++ b/docs/spec/v1alpha1/README.md @@ -0,0 +1,10 @@ +# source.fluxcd.io/v1alpha1 + +The is the v1alpha1 API specification for defining the desired state sources of Kubernetes clusters. + +Source kinds: +* [GitRepository](gitrepositories.md) +* [HelmRepository](helmrepositories.md) + +Implementations: +* source-controller [v0.0.1-alpha.1](https://github.com/fluxcd/source-controller/releases) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md new file mode 100644 index 00000000..eb8004e8 --- /dev/null +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -0,0 +1,212 @@ +# Git Repositories + +The `GitReposiory` API defines a source for artifacts coming from Git. + +## Specification + +Git repository spec: + +```go +// GitRepositorySpec defines the desired state of GitRepository +type GitRepositorySpec struct { + // +kubebuilder:validation:Pattern="^(http|https|ssh)://" + + // The repository URL, can be a HTTP or SSH address. + Url string `json:"url"` + + // The secret name containing the Git credentials + // +optional + SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` + + // The git reference to checkout and monitor for changes. + // +optional + Reference *GitRepositoryRef `json:"ref,omitempty"` + + // The interval at which to check for repository updates. + Interval metav1.Duration `json:"interval"` +} + +// GitRepositoryRef defines the git ref used for pull and checkout operations +type GitRepositoryRef struct { + // The git branch to checkout, defaults to ('master'). + // +optional + Branch string `json:"branch"` + + // The git tag to checkout, takes precedence over branch. + // +optional + Tag string `json:"tag"` + + // The git tag semver expression, takes precedence over tag. + // +optional + SemVer string `json:"semver"` +} +``` + +Git repository status: + +```go +// GitRepositoryStatus defines the observed state of GitRepository +type GitRepositoryStatus struct { + // +optional + Conditions []RepositoryCondition `json:"conditions,omitempty"` + + // LastUpdateTime is the timestamp corresponding to the last status + // change of this repository. + // +optional + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + + // URI for the artifacts of the last successful repository sync. + // +optional + Artifacts string `json:"artifacts,omitempty"` +} +``` + +Git repository status conditions: + +```go +// RepositoryCondition contains condition information for a repository +type RepositoryCondition struct { + // Type of the condition, currently ('Ready'). + Type RepositoryConditionType `json:"type"` + + // Status of the condition, one of ('True', 'False', 'Unknown'). + Status corev1.ConditionStatus `json:"status"` + + // LastTransitionTime is the timestamp corresponding to the last status + // change of this condition. + // +optional + LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` + + // Reason is a brief machine readable explanation for the condition's last + // transition. + // +optional + Reason string `json:"reason,omitempty"` + + // Message is a human readable description of the details of the last + // transition, complementing reason. + // +optional + Message string `json:"message,omitempty"` +} + +// RepositoryConditionType represents an repository condition value +type RepositoryConditionType string + +const ( + // RepositoryConditionReady represents the fact that a given repository condition + // is in ready state. + RepositoryConditionReady RepositoryConditionType = "Ready" +) +``` + +## Spec examples + +Public repository: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default + annotations: + # force sync trigger + source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master + tag: "3.2.0" + semver: ">= 3.2.0 <3.3.0" +``` + +HTTPS authentication: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + url: https://github.com/stefanprodan/podinfo + secretRef: + name: https-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-credentials + namespace: default +type: Opaque +data: + username: + password: +``` + +SSH authentication: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + url: ssh://git@github.com:stefanprodan/podinfo + secretRef: + name: ssh-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssh-credentials + namespace: default +type: Opaque +data: + identity: + identity.pub: + know_hosts: +``` + +Example of generating the SSH credentials secret: + +```bash +ssh-keygen -q -N "" -f ./identity +ssh-keyscan github.com > ./know_hosts + +kubectl create secret generic ssh-credentials \ + --from-file=./identity \ + --from-file=./identity.pub \ + --from-file=./know_hosts +``` + +## Status examples + +Successful sync: + +```yaml +status: + artifacts: http://source-controller.source-system/repositories/podinfo-default/5e747d3e088cd7a34ace4abc8cf7f3c3696e402f.tar.gz + conditions: + - lastTransitionTime: "2020-04-07T06:59:23Z" + message: 'Fetched artifacts are available at + /data/repositories/podinfo-default/5e747d3e088cd7a34ace4abc8cf7f3c3696e402f.tar.gz' + reason: GitCloneSucceed + status: "True" + type: Ready + lastUpdateTime: "2020-04-07T06:59:23Z" +``` + +Failed sync: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-06T06:48:59Z" + message: 'git clone error ssh: handshake failed: ssh: unable to authenticate, + attempted methods [none publickey], no supported methods remain' + reason: GitCloneFailed + status: "False" + type: Ready +``` diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md new file mode 100644 index 00000000..4e71d089 --- /dev/null +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -0,0 +1,15 @@ +# Helm Repositories + +The `HelmReposiory` and `HelmChart` API defines a source for artifacts coming from Helm repositories. + +## Specification + +TODO + +## Spec examples + +TODO + +## Status examples + +TODO \ No newline at end of file From 424c1cc0972303d0642a8dbfbb04575df2b42746 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 Apr 2020 13:07:36 +0300 Subject: [PATCH 02/21] Add commit pinning option to Git repository API --- docs/spec/v1alpha1/gitrepositories.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index eb8004e8..ab6e85f4 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -39,6 +39,10 @@ type GitRepositoryRef struct { // The git tag semver expression, takes precedence over tag. // +optional SemVer string `json:"semver"` + + // The git commit sha to checkout, if specified branch and tag filters will be ignored. + // +optional + Commit string `json:"commit"` } ``` From 5ca2f9a4c9321ab5cfb7e11057b06901064599a5 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 Apr 2020 13:09:03 +0300 Subject: [PATCH 03/21] Fix known_hosts typo in spec --- docs/spec/v1alpha1/gitrepositories.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index ab6e85f4..080c764b 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -170,19 +170,19 @@ type: Opaque data: identity: identity.pub: - know_hosts: + known_hosts: ``` Example of generating the SSH credentials secret: ```bash ssh-keygen -q -N "" -f ./identity -ssh-keyscan github.com > ./know_hosts +ssh-keyscan github.com > ./known_hosts kubectl create secret generic ssh-credentials \ --from-file=./identity \ --from-file=./identity.pub \ - --from-file=./know_hosts + --from-file=./known_hosts ``` ## Status examples From 6a1d0409cbbf762122009cceb2d6eb426df0003c Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 Apr 2020 13:11:37 +0300 Subject: [PATCH 04/21] Change the manifests transformation scope --- docs/spec/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/spec/README.md b/docs/spec/README.md index 4e43ca52..d8979524 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -23,11 +23,11 @@ Decoupling the Kubernetes objects acquisition from the reconciliation process wi source manager would make Flux and Helm Operator resilient to outbound connectivity issues and would simplify the state machine(s) that these controllers operate. -Managing the source operations in a dedicated controller could allow Flux to compose the desire state of a cluster +Managing the source operations in a dedicated controller could enable Flux to compose the desire state of a cluster from multiple source. Further more the manifests transformation process could be performed by 3rd party tools (e.g. kustomize, jk, tanka, cue run by Tekton pipelines or Kubernetes Jobs) -that would output the final state of the Kubernetes objects into an artifacts repository maintained by the source controller. +that subscribe to source changes events. ## Goals From a9a73301dd0eaaa74b54843180a49816569a2b78 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Thu, 9 Apr 2020 13:30:22 +0300 Subject: [PATCH 05/21] Fix typos Co-Authored-By: Michael Bridgen --- docs/spec/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/spec/README.md b/docs/spec/README.md index d8979524..9024044a 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -17,7 +17,7 @@ private HTTPS servers. Each Flux or Helm Operator instance maintains its own Git repository mirror even if all of them point to the same source. If the Git repository host becomes unavailable, the cluster state will diverge from the last -know desired state since the operators will stop the reconciliation due to pull errors. +known desired state since the operators will stop the reconciliation due to pull errors. Decoupling the Kubernetes objects acquisition from the reconciliation process with an in-cluster source manager would make Flux and Helm Operator resilient to outbound connectivity issues and would @@ -25,7 +25,7 @@ simplify the state machine(s) that these controllers operate. Managing the source operations in a dedicated controller could enable Flux to compose the desire state of a cluster from multiple source. -Further more the manifests transformation process could be performed by 3rd party tools +Furthermore, the manifests transformation process could be performed by 3rd party tools (e.g. kustomize, jk, tanka, cue run by Tekton pipelines or Kubernetes Jobs) that subscribe to source changes events. From 134633266ad56f59b388e7814af24a432b4df30f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 9 Apr 2020 15:28:07 +0300 Subject: [PATCH 06/21] Add semver automation motive --- docs/spec/README.md | 9 +++++++-- docs/spec/v1alpha1/gitrepositories.md | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/spec/README.md b/docs/spec/README.md index 9024044a..fe998f6c 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -24,11 +24,16 @@ source manager would make Flux and Helm Operator resilient to outbound connectiv simplify the state machine(s) that these controllers operate. Managing the source operations in a dedicated controller could enable Flux to compose the desire state of a cluster -from multiple source. -Furthermore, the manifests transformation process could be performed by 3rd party tools +from multiple source. Furthermore, the manifests transformation process could be performed by 3rd party tools (e.g. kustomize, jk, tanka, cue run by Tekton pipelines or Kubernetes Jobs) that subscribe to source changes events. +The source controller could enable pinning the cluster desired state to a specific Git commit or Git tag. + +For teams that are using semantic versioning, the source controller could monitor the Git repository tags +and set the cluster desired state to the latest release or to a tag that matches a semver pattern. +In a similar fashion, a semver pattern could trigger Helm chart upgrades without manual intervention from users. + ## Goals The main goal is to define a set of Kubernetes objects that cluster admins and various automated operators diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 080c764b..18df0b02 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -18,7 +18,7 @@ type GitRepositorySpec struct { // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` - // The git reference to checkout and monitor for changes. + // The git reference to checkout and monitor for changes, defaults to master branch. // +optional Reference *GitRepositoryRef `json:"ref,omitempty"` @@ -28,7 +28,7 @@ type GitRepositorySpec struct { // GitRepositoryRef defines the git ref used for pull and checkout operations type GitRepositoryRef struct { - // The git branch to checkout, defaults to ('master'). + // The git branch to checkout, defaults to master. // +optional Branch string `json:"branch"` From 52a2ae80d13572cb70aaa2d5255c2119765bf6e9 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 10 Apr 2020 11:44:46 +0200 Subject: [PATCH 07/21] Draft HelmRepository and HelmRelease specs --- docs/spec/v1alpha1/README.md | 9 +- docs/spec/v1alpha1/common.md | 62 ++++++ docs/spec/v1alpha1/helmrepositories.md | 280 ++++++++++++++++++++++++- 3 files changed, 344 insertions(+), 7 deletions(-) create mode 100644 docs/spec/v1alpha1/common.md diff --git a/docs/spec/v1alpha1/README.md b/docs/spec/v1alpha1/README.md index 23491974..f1e49e62 100644 --- a/docs/spec/v1alpha1/README.md +++ b/docs/spec/v1alpha1/README.md @@ -2,9 +2,12 @@ The is the v1alpha1 API specification for defining the desired state sources of Kubernetes clusters. -Source kinds: -* [GitRepository](gitrepositories.md) -* [HelmRepository](helmrepositories.md) + +* [Common](common.md) +* Source kinds: + + [GitRepository](gitrepositories.md) + + [HelmRepository](helmrepositories.md) + - [HelmChart](helmrepositories.md) Implementations: * source-controller [v0.0.1-alpha.1](https://github.com/fluxcd/source-controller/releases) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md new file mode 100644 index 00000000..80767e2d --- /dev/null +++ b/docs/spec/v1alpha1/common.md @@ -0,0 +1,62 @@ +# Common + +Common defines resources used across types. + +## Specification + +### Source condition + +> **Note:** to be replaced with +> once made available. + +```go +// SourceCondition contains condition information for a source. +type SourceCondition struct { + // Type of the condition, currently ('Ready'). + // +required + Type string `json:"type"` + + // Status of the condition, one of ('True', 'False', 'Unknown'). + // +required + Status corev1.ConditionStatus `json:"status"` + + // LastTransitionTime is the timestamp corresponding to the last status + // change of this condition. + // +required + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // Reason is a brief machine readable explanation for the condition's last + // transition. + // +required + Reason string `json:"reason,omitempty"` + + // Message is a human readable description of the details of the last + // transition, complementing reason. + // +optional + Message string `json:"message,omitempty"` +} +``` + +#### Types + +```go +const ( + // ReadyCondition represents the fact that a given source is in ready state. + ReadyCondition string = "Ready" +) +``` + +#### Reasons + +```go +const ( + // InitializingReason represents the fact that a given source is being initialized. + InitializingReason string = "Initializing" + // URLInvalidReason represents the fact that a given source has an invalid URL. + URLInvalidReason string = "URLInvalid" +) +``` + +## Examples + +See the [Git repository](gitrepositories.md) and [Helm chart](helmrepositories.md) APIs. diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index 4e71d089..8d7e381e 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -1,15 +1,287 @@ # Helm Repositories -The `HelmReposiory` and `HelmChart` API defines a source for artifacts coming from Helm repositories. +The Helm source API defines two sources for artifact coming from Helm: +`HelmRepository` and `HelmChart`. ## Specification -TODO +### Helm repository + +```go +// HelmRepository defines the reference to a Helm repository. +type HelmRepositorySpec struct { + // The Helm repository URL, a valid URL contains at least a + // protocol and host. + // +required + URL string `json:"url"` + + // The name of the secret containing authentication credentials + // for the Helm repository. + // +optional + SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` + + // The interval at which to check the upstream for updates. + // +required + Interval metav1.Duration `json:"interval"` +} +``` + +#### Helm repository status + +```go +// HelmRepositoryStatus defines the observed state of the HelmRepository. +type HelmRepositoryStatus struct { + // +optional + Conditions []SourceCondition `json:"conditions,omitempty"` + + // LastUpdateTime is the timestamp corresponding to the last status + // change of this Helm repository. + // +optional + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + + // URI for the artifact of the last successful repository index. + // +optional + Artifact string `json:"artifact,omitempty"` +} +``` + +#### Helm repository condition reasons + +```go +const ( + // IndexationFailedReason represents the fact that the indexation + // of the given Helm repository failed. + IndexationFailedReason string = "IndexationFailed" + // IndexationSucceededReason represents the fact that the indexation + // of the given Helm repository succeeded. + IndexationSucceedReason string = "IndexationSucceed" +) +``` + +### Helm chart + +```go +// HelmChart defines the desired state of a Helm chart. +type HelmChartSpec struct { + // The name of the Helm chart, as made available by the referenced + // Helm repository. + // +required + Name string `json:"name"` + + // The chart version semver expression, defaults to latest when + // omitted. + // +optional + Version string `json:"version,omitempty"` + + // The name of the HelmRepository the chart is available at. + // +required + HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` + + // The interval at which to check the Helm repository for updates. + // Defaults to the interval of the Helm repository. + // +optional + Interval metav1.Duration `json:"interval,omitempty"` +} +``` + +#### Helm chart status + +```go +// HelmChartStatus defines the observed state of the HelmChart. +type HelmChartStatus struct { + // +optional + Conditions []SourceCondition `json:"conditions,omitempty"` + + // LastUpdateTime is the timestamp corresponding to the last status + // change of this Helm chart. + // +optional + LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + + // URI for the artifact of the latest successful Helm chart pull. + // +optional + Artifact string `json:"artifact,omitempty"` +} +``` + +#### Helm chart condition reasons + +```go +const ( + // ChartPullFailedReason represents the fact that the pull of the + // given Helm chart failed. + ChartPullFailedReason string = "ChartPullFailed" + // ChartPullSucceededReason represents the fact that the pull of + // the given Helm chart succeeded. + ChartPullSucceedReason string = "ChartPullSucceeded" +) +``` ## Spec examples -TODO +### Helm repository + +Public Helm repository: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmRepository +metadata: + name: stable + namespace: default + annotations: + # force sync trigger + source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" +spec: + url: https://kubernetes-charts.storage.googleapis.com/ + interval: 1m +``` + +Private Helm repository: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmRepository +metadata: + name: private + namespace: default +spec: + url: https://charts.example.com + secretRef: + name: https-credentials + interval: 1m +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-credentials + namespace: default +type: Opaque +data: + username: + password: + certFile: + keyFile: + caFile: + insecureSkipTLSVerify: +``` + +### Helm chart + +Pinned version: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default + annotations: + # force sync trigger + source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" +spec: + name: redis + version: 10.5.7 + helmRepositoryRef: + name: stable +``` + +Semver range: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default +spec: + name: redis + version: ^10.0.0 + helmRepositoryRef: + name: stable +``` + +Interval: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default +spec: + name: redis + version: ^10.0.0 + helmRepositoryRef: + name: stable + interval: 30m +``` ## Status examples -TODO \ No newline at end of file +### Helm repository + +Successful indexation: + +```yaml +status: + artifact: http:///helmrepositories/podinfo-default/index-21c195d78e699e4b656e2885887d019627838993.yaml + conditions: + - lastTransitionTime: "2020-04-10T09:34:45Z" + message: Fetched artifact are available at /data/helmrepositories/podinfo-default/index-21c195d78e699e4b656e2885887d019627838993.yaml + reason: IndexationSucceeded + status: "True" + type: Ready + lastUpdateTime: "2020-04-10T09:34:45Z" +``` + +Failed indexation: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-10T09:27:21Z" + message: 'failed to fetch https://invalid.example.com/index.yaml : 404 Not Found' + reason: IndexationFailed + status: "False" + type: Ready +``` + +Invalid repository URL: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-10T09:27:21Z" + message: scheme "invalid" not supported + reason: URLInvalid + status: "False" + type: Ready +``` + +### Helm chart + +Successful chart pull: + +```yaml +status: + artifact: http:///helmcharts/redis-default/redis-10.5.7.tgz + conditions: + - lastTransitionTime: "2020-04-10T09:34:45Z" + message: Fetched artifact are available at /data/helmcharts/redis-default/redis-10.5.7.tgz + reason: ChartPullSucceeded + status: "True" + type: Ready + lastUpdateTime: "2020-04-10T09:34:45Z" +``` + +Failed chart pull: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-10T09:34:45Z" + message: '' + reason: ChartPullFailed + status: "False" + type: Ready +``` From 1e5765ba4fff05db7ff4a421d98c971cf3015603 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 10 Apr 2020 21:37:13 +0300 Subject: [PATCH 08/21] Add condition reasons and auth required fields --- docs/spec/v1alpha1/common.md | 4 ++ docs/spec/v1alpha1/gitrepositories.md | 69 ++++++++++----------------- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 80767e2d..1a474fc6 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -54,6 +54,10 @@ const ( InitializingReason string = "Initializing" // URLInvalidReason represents the fact that a given source has an invalid URL. URLInvalidReason string = "URLInvalid" + // StorageOperationFailedReason signals a failure caused by a storage operation. + StorageOperationFailedReason string = "StorageOperationFailed" + // AuthenticationFailedReason represents the fact that the provided credentials are not valid. + AuthenticationFailedReason string = "AuthenticationFailed" ) ``` diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 18df0b02..35121192 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -4,15 +4,14 @@ The `GitReposiory` API defines a source for artifacts coming from Git. ## Specification -Git repository spec: +Git repository: ```go // GitRepositorySpec defines the desired state of GitRepository type GitRepositorySpec struct { - // +kubebuilder:validation:Pattern="^(http|https|ssh)://" - // The repository URL, can be a HTTP or SSH address. - Url string `json:"url"` + // +kubebuilder:validation:Pattern="^(http|https|ssh)://" + URL string `json:"url"` // The secret name containing the Git credentials // +optional @@ -25,7 +24,11 @@ type GitRepositorySpec struct { // The interval at which to check for repository updates. Interval metav1.Duration `json:"interval"` } +``` +Git repository reference: + +```go // GitRepositoryRef defines the git ref used for pull and checkout operations type GitRepositoryRef struct { // The git branch to checkout, defaults to master. @@ -46,65 +49,41 @@ type GitRepositoryRef struct { } ``` -Git repository status: +#### Status ```go // GitRepositoryStatus defines the observed state of GitRepository type GitRepositoryStatus struct { // +optional - Conditions []RepositoryCondition `json:"conditions,omitempty"` + Conditions []SourceCondition `json:"conditions,omitempty"` // LastUpdateTime is the timestamp corresponding to the last status // change of this repository. // +optional LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` - // URI for the artifacts of the last successful repository sync. + // Path to the artifact output of the last repository sync. // +optional - Artifacts string `json:"artifacts,omitempty"` + Artifact string `json:"artifacts,omitempty"` } ``` -Git repository status conditions: +#### Condition reasons ```go -// RepositoryCondition contains condition information for a repository -type RepositoryCondition struct { - // Type of the condition, currently ('Ready'). - Type RepositoryConditionType `json:"type"` - - // Status of the condition, one of ('True', 'False', 'Unknown'). - Status corev1.ConditionStatus `json:"status"` - - // LastTransitionTime is the timestamp corresponding to the last status - // change of this condition. - // +optional - LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` - - // Reason is a brief machine readable explanation for the condition's last - // transition. - // +optional - Reason string `json:"reason,omitempty"` - - // Message is a human readable description of the details of the last - // transition, complementing reason. - // +optional - Message string `json:"message,omitempty"` -} - -// RepositoryConditionType represents an repository condition value -type RepositoryConditionType string - const ( - // RepositoryConditionReady represents the fact that a given repository condition - // is in ready state. - RepositoryConditionReady RepositoryConditionType = "Ready" + // GitOperationSucceedReason represents the fact that the git + // clone, pull and checkout operations succeeded. + GitOperationSucceedReason string = "GitOperationSucceed" + // GitOperationFailedReason represents the fact that the git + // clone, pull or checkout operations failed. + GitOperationFailedReason string = "GitOperationFailed" ) ``` ## Spec examples -Public repository: +Public Git repository: ```yaml apiVersion: source.fluxcd.io/v1alpha1 @@ -113,7 +92,7 @@ metadata: name: podinfo namespace: default annotations: - # force sync trigger + # on-demand sync trigger source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" spec: interval: 1m @@ -124,7 +103,7 @@ spec: semver: ">= 3.2.0 <3.3.0" ``` -HTTPS authentication: +HTTPS authentication (requires a secret with `username` and `password` fields): ```yaml apiVersion: source.fluxcd.io/v1alpha1 @@ -148,7 +127,7 @@ data: password: ``` -SSH authentication: +SSH authentication (requires a secret with `identity` and `known_hosts` fields): ```yaml apiVersion: source.fluxcd.io/v1alpha1 @@ -196,7 +175,7 @@ status: - lastTransitionTime: "2020-04-07T06:59:23Z" message: 'Fetched artifacts are available at /data/repositories/podinfo-default/5e747d3e088cd7a34ace4abc8cf7f3c3696e402f.tar.gz' - reason: GitCloneSucceed + reason: GitOperationSucceed status: "True" type: Ready lastUpdateTime: "2020-04-07T06:59:23Z" @@ -210,7 +189,7 @@ status: - lastTransitionTime: "2020-04-06T06:48:59Z" message: 'git clone error ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain' - reason: GitCloneFailed + reason: AuthenticationFailed status: "False" type: Ready ``` From fee4c261c6f716affdd9148e6d713bf5fa305705 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 12 Apr 2020 14:07:23 +0300 Subject: [PATCH 09/21] Add artifact type to source status API --- docs/spec/v1alpha1/common.md | 30 ++++++++++- docs/spec/v1alpha1/gitrepositories.md | 71 +++++++++++++++++++++----- docs/spec/v1alpha1/helmrepositories.md | 28 +++++----- 3 files changed, 100 insertions(+), 29 deletions(-) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 1a474fc6..3b80ccc5 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -4,6 +4,34 @@ Common defines resources used across types. ## Specification +### Source status + +Source objects should contain a status sub-resource that embeds an artifact object: + +```go +// Artifact represents the output of a source synchronisation +type Artifact struct { + // Path is the local file path of this artifact. + // +required + Path string `json:"path"` + + // URL is the HTTP address of this artifact. + // +required + URL string `json:"url"` + + // Revision is a human readable identifier traceable in the origin source system. + // It can be a commit sha, git tag, a helm index timestamp, + // a helm chart version, a checksum, etc. + // +optional + Revision string `json:"revision"` + + // LastUpdateTime is the timestamp corresponding to the last + // update of this artifact. + // +required + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` +} +``` + ### Source condition > **Note:** to be replaced with @@ -52,7 +80,7 @@ const ( const ( // InitializingReason represents the fact that a given source is being initialized. InitializingReason string = "Initializing" - // URLInvalidReason represents the fact that a given source has an invalid URL. + // URLInvalidReason represents the fact that a given source has an invalid URL. URLInvalidReason string = "URLInvalid" // StorageOperationFailedReason signals a failure caused by a storage operation. StorageOperationFailedReason string = "StorageOperationFailed" diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 35121192..ebc9293e 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -57,14 +57,13 @@ type GitRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` - // LastUpdateTime is the timestamp corresponding to the last status - // change of this repository. + // URL is the download link for the artifact output of the last repository sync. // +optional - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + URL string `json:"url,omitempty"` - // Path to the artifact output of the last repository sync. + // Artifact represents the output of the last successful repository sync. // +optional - Artifact string `json:"artifacts,omitempty"` + Artifact *Artifact `json:"artifact,omitempty"` } ``` @@ -83,7 +82,7 @@ const ( ## Spec examples -Public Git repository: +Pull the master of a public repository every minute: ```yaml apiVersion: source.fluxcd.io/v1alpha1 @@ -99,8 +98,52 @@ spec: url: https://github.com/stefanprodan/podinfo ref: branch: master - tag: "3.2.0" - semver: ">= 3.2.0 <3.3.0" +``` + +Checkout a specific commit from a branch: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master + commit: 363a6a8fe6a7f13e05d34c163b0ef02a777da20a +``` + +Pull a specific tag: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + tag: 3.2.0 +``` + +Pull tag based on semver expression: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + semver: ">=3.1.0-rc.1 <3.2.0" ``` HTTPS authentication (requires a secret with `username` and `password` fields): @@ -136,7 +179,7 @@ metadata: name: podinfo namespace: default spec: - url: ssh://git@github.com:stefanprodan/podinfo + url: ssh://git@github.com/stefanprodan/podinfo secretRef: name: ssh-credentials --- @@ -170,15 +213,19 @@ Successful sync: ```yaml status: - artifacts: http://source-controller.source-system/repositories/podinfo-default/5e747d3e088cd7a34ace4abc8cf7f3c3696e402f.tar.gz + artifact: + lastUpdateTime: "2020-04-07T06:59:23Z" + path: /data/gitrepository/podinfo-default/363a6a8fe6a7f13e05d34c163b0ef02a777da20a.tar.gz + revision: master/363a6a8fe6a7f13e05d34c163b0ef02a777da20a + url: http:///gitrepository/podinfo-default/363a6a8fe6a7f13e05d34c163b0ef02a777da20a.tar.gz conditions: - lastTransitionTime: "2020-04-07T06:59:23Z" message: 'Fetched artifacts are available at - /data/repositories/podinfo-default/5e747d3e088cd7a34ace4abc8cf7f3c3696e402f.tar.gz' + /data/gitrepository/podinfo-default/363a6a8fe6a7f13e05d34c163b0ef02a777da20a.tar.gz' reason: GitOperationSucceed status: "True" type: Ready - lastUpdateTime: "2020-04-07T06:59:23Z" + url: http:///gitrepository/podinfo-default/latest.tar.gz ``` Failed sync: diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index 8d7e381e..9ab550fb 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -29,19 +29,18 @@ type HelmRepositorySpec struct { #### Helm repository status ```go -// HelmRepositoryStatus defines the observed state of the HelmRepository. +// HelmRepositoryStatus defines the observed state of HelmRepository type HelmRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` - // LastUpdateTime is the timestamp corresponding to the last status - // change of this Helm repository. + // URL is the download link for the last index fetched. // +optional - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + URL string `json:"url,omitempty"` - // URI for the artifact of the last successful repository index. + // Artifact represents the output of the last successful repository sync. // +optional - Artifact string `json:"artifact,omitempty"` + Artifact *Artifact `json:"artifact,omitempty"` } ``` @@ -88,18 +87,17 @@ type HelmChartSpec struct { ```go // HelmChartStatus defines the observed state of the HelmChart. -type HelmChartStatus struct { +type HelmRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` - // LastUpdateTime is the timestamp corresponding to the last status - // change of this Helm chart. + // URL is the download link for the last chart fetched. // +optional - LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"` + URL string `json:"url,omitempty"` - // URI for the artifact of the latest successful Helm chart pull. + // Artifact represents the output of the last successful sync. // +optional - Artifact string `json:"artifact,omitempty"` + Artifact *Artifact `json:"artifact,omitempty"` } ``` @@ -224,14 +222,13 @@ Successful indexation: ```yaml status: - artifact: http:///helmrepositories/podinfo-default/index-21c195d78e699e4b656e2885887d019627838993.yaml + url: http:///helmrepository/podinfo-default/index.yaml conditions: - lastTransitionTime: "2020-04-10T09:34:45Z" message: Fetched artifact are available at /data/helmrepositories/podinfo-default/index-21c195d78e699e4b656e2885887d019627838993.yaml reason: IndexationSucceeded status: "True" type: Ready - lastUpdateTime: "2020-04-10T09:34:45Z" ``` Failed indexation: @@ -264,14 +261,13 @@ Successful chart pull: ```yaml status: - artifact: http:///helmcharts/redis-default/redis-10.5.7.tgz + url: http:///helmcharts/redis-default/redis-10.5.7.tgz conditions: - lastTransitionTime: "2020-04-10T09:34:45Z" message: Fetched artifact are available at /data/helmcharts/redis-default/redis-10.5.7.tgz reason: ChartPullSucceeded status: "True" type: Ready - lastUpdateTime: "2020-04-10T09:34:45Z" ``` Failed chart pull: From 44a5f21f3bef24c5a03076d9c621e3c911fca72b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 12 Apr 2020 14:27:36 +0300 Subject: [PATCH 10/21] Add link to semver ranges --- docs/spec/v1alpha1/gitrepositories.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index ebc9293e..37e7c2ec 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -82,7 +82,7 @@ const ( ## Spec examples -Pull the master of a public repository every minute: +Pull the master branch of a public repository every minute: ```yaml apiVersion: source.fluxcd.io/v1alpha1 @@ -93,11 +93,24 @@ metadata: annotations: # on-demand sync trigger source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo +``` + +Pull a specific branch: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default spec: interval: 1m url: https://github.com/stefanprodan/podinfo ref: - branch: master + branch: v3.x ``` Checkout a specific commit from a branch: @@ -131,7 +144,7 @@ spec: tag: 3.2.0 ``` -Pull tag based on semver expression: +Pull tag based on a [semver range](https://github.com/blang/semver#ranges): ```yaml apiVersion: source.fluxcd.io/v1alpha1 From 6baaa34b0a95ebe72b8ed04c2fb171809b554691 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 13 Apr 2020 13:14:42 +0300 Subject: [PATCH 11/21] Add source synchronization to common spec --- docs/spec/v1alpha1/common.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 3b80ccc5..1b6d44e1 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -1,9 +1,38 @@ # Common -Common defines resources used across types. +Common defines resources used across all source types. ## Specification +### Source synchronization + +Source objects should contain a `spec.interval` field that tells the controller at which interval to check for updates: + +```go +type SourceSpec struct { + // The interval at which to check for source updates. + // +required + Interval metav1.Duration `json:"interval"` +} +``` + +Valid time units are `s`, `m` and `h` e.g. `interval: 5m`. + +The controller can be told to check for updates right away by setting an annotation on source objects: + +```go +const ( + // ForceSyncAnnotation is the timestamp corresponding to an on-demand source sync. + ForceSyncAnnotation string = "source.fluxcd.io/syncAt" +) +``` + +Force sync example: + +```bash +kubectl annotate --overwrite gitrepository/podinfo source.fluxcd.io/syncAt="$(date +%s)" +``` + ### Source status Source objects should contain a status sub-resource that embeds an artifact object: From a490b25647e7abd2fcbd8d5d999c789775fbe099 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 13 Apr 2020 13:28:24 +0300 Subject: [PATCH 12/21] Add wait for condition example --- docs/spec/v1alpha1/gitrepositories.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 37e7c2ec..5c01a3a4 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -13,7 +13,7 @@ type GitRepositorySpec struct { // +kubebuilder:validation:Pattern="^(http|https|ssh)://" URL string `json:"url"` - // The secret name containing the Git credentials + // The secret name containing the Git credentials. // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` @@ -90,9 +90,6 @@ kind: GitRepository metadata: name: podinfo namespace: default - annotations: - # on-demand sync trigger - source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" spec: interval: 1m url: https://github.com/stefanprodan/podinfo @@ -208,6 +205,9 @@ data: known_hosts: ``` +> **Note:** that the SSH address does not support SCP syntax. The URL format is +> `ssh://user@host:port/org/repository`. + Example of generating the SSH credentials secret: ```bash @@ -253,3 +253,9 @@ status: status: "False" type: Ready ``` + +Wait for condition: + +```bash +kubectl wait gitrepository/podinfo --for=condition=ready --timeout=1m +``` From 086f6a350a6c2ff3eb1a8ef19b4aa29ff16fc939 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 13 Apr 2020 13:47:32 +0300 Subject: [PATCH 13/21] Remove insecure TLS option for Helm spec --- docs/spec/v1alpha1/README.md | 4 +++- docs/spec/v1alpha1/common.md | 2 +- docs/spec/v1alpha1/helmrepositories.md | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/spec/v1alpha1/README.md b/docs/spec/v1alpha1/README.md index f1e49e62..eb9bf58b 100644 --- a/docs/spec/v1alpha1/README.md +++ b/docs/spec/v1alpha1/README.md @@ -2,6 +2,7 @@ The is the v1alpha1 API specification for defining the desired state sources of Kubernetes clusters. +## Specification * [Common](common.md) * Source kinds: @@ -9,5 +10,6 @@ The is the v1alpha1 API specification for defining the desired state sources of + [HelmRepository](helmrepositories.md) - [HelmChart](helmrepositories.md) -Implementations: +## Implementation + * source-controller [v0.0.1-alpha.1](https://github.com/fluxcd/source-controller/releases) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 1b6d44e1..092bcec0 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -22,7 +22,7 @@ The controller can be told to check for updates right away by setting an annotat ```go const ( - // ForceSyncAnnotation is the timestamp corresponding to an on-demand source sync. + // ForceSyncAnnotation is the timestamp corresponding to an on-demand source sync. ForceSyncAnnotation string = "source.fluxcd.io/syncAt" ) ``` diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index 9ab550fb..a52b0265 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -160,7 +160,6 @@ data: certFile: keyFile: caFile: - insecureSkipTLSVerify: ``` ### Helm chart From 55a6d5811ee709bac167cee14a31c8c705663ccb Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 13 Apr 2020 17:16:19 +0200 Subject: [PATCH 14/21] Move HelmChart spec to individual document --- docs/spec/v1alpha1/README.md | 2 +- docs/spec/v1alpha1/gitrepositories.md | 23 ++-- docs/spec/v1alpha1/helmcharts.md | 142 ++++++++++++++++++++++ docs/spec/v1alpha1/helmrepositories.md | 160 ++----------------------- 4 files changed, 168 insertions(+), 159 deletions(-) create mode 100644 docs/spec/v1alpha1/helmcharts.md diff --git a/docs/spec/v1alpha1/README.md b/docs/spec/v1alpha1/README.md index eb9bf58b..7ecb44df 100644 --- a/docs/spec/v1alpha1/README.md +++ b/docs/spec/v1alpha1/README.md @@ -8,7 +8,7 @@ The is the v1alpha1 API specification for defining the desired state sources of * Source kinds: + [GitRepository](gitrepositories.md) + [HelmRepository](helmrepositories.md) - - [HelmChart](helmrepositories.md) + + [HelmChart](helmcharts.md) ## Implementation diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 5c01a3a4..945fe6e8 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -1,13 +1,15 @@ # Git Repositories -The `GitReposiory` API defines a source for artifacts coming from Git. +The `GitReposiory` API defines a source for artifacts coming from Git. The +resource exposes the latest synchronized state from Git as an artifact in +an archive. ## Specification Git repository: ```go -// GitRepositorySpec defines the desired state of GitRepository +// GitRepositorySpec defines the desired state of GitRepository. type GitRepositorySpec struct { // The repository URL, can be a HTTP or SSH address. // +kubebuilder:validation:Pattern="^(http|https|ssh)://" @@ -17,7 +19,8 @@ type GitRepositorySpec struct { // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` - // The git reference to checkout and monitor for changes, defaults to master branch. + // The git reference to checkout and monitor for changes, defaults to + // master branch. // +optional Reference *GitRepositoryRef `json:"ref,omitempty"` @@ -29,7 +32,7 @@ type GitRepositorySpec struct { Git repository reference: ```go -// GitRepositoryRef defines the git ref used for pull and checkout operations +// GitRepositoryRef defines the git ref used for pull and checkout operations. type GitRepositoryRef struct { // The git branch to checkout, defaults to master. // +optional @@ -43,21 +46,23 @@ type GitRepositoryRef struct { // +optional SemVer string `json:"semver"` - // The git commit sha to checkout, if specified branch and tag filters will be ignored. + // The git commit sha to checkout, if specified branch and tag filters will + // be ignored. // +optional Commit string `json:"commit"` } ``` -#### Status +### Status ```go -// GitRepositoryStatus defines the observed state of GitRepository +// GitRepositoryStatus defines the observed state of GitRepository. type GitRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` - // URL is the download link for the artifact output of the last repository sync. + // URL is the download link for the artifact output of the last repository + // sync. // +optional URL string `json:"url,omitempty"` @@ -67,7 +72,7 @@ type GitRepositoryStatus struct { } ``` -#### Condition reasons +### Condition reasons ```go const ( diff --git a/docs/spec/v1alpha1/helmcharts.md b/docs/spec/v1alpha1/helmcharts.md new file mode 100644 index 00000000..2a27101a --- /dev/null +++ b/docs/spec/v1alpha1/helmcharts.md @@ -0,0 +1,142 @@ +# Helm Charts + +The `HelmChart` API defines a source for Helm chart artifacts coming +from [`HelmRepository` sources](helmrepositories.md). The resource +exposes the latest pulled chart for the defined version as an artifact. + +## Specification + +Helm chart: + +```go +// HelmChart defines the desired state of a Helm chart. +type HelmChartSpec struct { + // The name of the Helm chart, as made available by the referenced + // Helm repository. + // +required + Name string `json:"name"` + + // The chart version semver expression, defaults to latest when + // omitted. + // +optional + Version string `json:"version,omitempty"` + + // The name of the HelmRepository the chart is available at. + // +required + HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` + + // The interval at which to check the Helm repository for updates. + // Defaults to the interval of the referenced HelmRepository. + // +optional + Interval metav1.Duration `json:"interval,omitempty"` +} +``` + +### Status + +```go +// HelmChartStatus defines the observed state of the HelmChart. +type HelmRepositoryStatus struct { + // +optional + Conditions []SourceCondition `json:"conditions,omitempty"` + + // URL is the download link for the last chart fetched. + // +optional + URL string `json:"url,omitempty"` + + // Artifact represents the output of the last successful chart sync. + // +optional + Artifact *Artifact `json:"artifact,omitempty"` +} +``` + +### Condition reasons + +```go +const ( + // ChartPullFailedReason represents the fact that the pull of the + // given Helm chart failed. + ChartPullFailedReason string = "ChartPullFailed" + // ChartPullSucceededReason represents the fact that the pull of + // the given Helm chart succeeded. + ChartPullSucceedReason string = "ChartPullSucceeded" +) +``` + +## Spec examples + +Pinned version: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default + annotations: + # force sync trigger + source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" +spec: + name: redis + version: 10.5.7 + helmRepositoryRef: + name: stable +``` + +Semver range: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default +spec: + name: redis + version: ^10.0.0 + helmRepositoryRef: + name: stable +``` + +Interval: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: redis + namespace: default +spec: + name: redis + version: ^10.0.0 + helmRepositoryRef: + name: stable + interval: 30m +``` + +## Status examples + +Successful chart pull: + +```yaml +status: + url: http:///helmcharts/redis-default/redis-10.5.7.tgz + conditions: + - lastTransitionTime: "2020-04-10T09:34:45Z" + message: Fetched artifact are available at /data/helmcharts/redis-default/redis-10.5.7.tgz + reason: ChartPullSucceeded + status: "True" + type: Ready +``` + +Failed chart pull: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-10T09:34:45Z" + message: '' + reason: ChartPullFailed + status: "False" + type: Ready +``` diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index a52b0265..851a3ac1 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -1,11 +1,12 @@ # Helm Repositories -The Helm source API defines two sources for artifact coming from Helm: -`HelmRepository` and `HelmChart`. +The `HelmRepository` API defines a source for Helm repositories. +The resource exposes the latest synchronized repository index as +an artifact. ## Specification -### Helm repository +Helm repository: ```go // HelmRepository defines the reference to a Helm repository. @@ -26,7 +27,7 @@ type HelmRepositorySpec struct { } ``` -#### Helm repository status +### Status ```go // HelmRepositoryStatus defines the observed state of HelmRepository @@ -44,7 +45,7 @@ type HelmRepositoryStatus struct { } ``` -#### Helm repository condition reasons +### Condition reasons ```go const ( @@ -57,67 +58,8 @@ const ( ) ``` -### Helm chart - -```go -// HelmChart defines the desired state of a Helm chart. -type HelmChartSpec struct { - // The name of the Helm chart, as made available by the referenced - // Helm repository. - // +required - Name string `json:"name"` - - // The chart version semver expression, defaults to latest when - // omitted. - // +optional - Version string `json:"version,omitempty"` - - // The name of the HelmRepository the chart is available at. - // +required - HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` - - // The interval at which to check the Helm repository for updates. - // Defaults to the interval of the Helm repository. - // +optional - Interval metav1.Duration `json:"interval,omitempty"` -} -``` - -#### Helm chart status - -```go -// HelmChartStatus defines the observed state of the HelmChart. -type HelmRepositoryStatus struct { - // +optional - Conditions []SourceCondition `json:"conditions,omitempty"` - - // URL is the download link for the last chart fetched. - // +optional - URL string `json:"url,omitempty"` - - // Artifact represents the output of the last successful sync. - // +optional - Artifact *Artifact `json:"artifact,omitempty"` -} -``` - -#### Helm chart condition reasons - -```go -const ( - // ChartPullFailedReason represents the fact that the pull of the - // given Helm chart failed. - ChartPullFailedReason string = "ChartPullFailed" - // ChartPullSucceededReason represents the fact that the pull of - // the given Helm chart succeeded. - ChartPullSucceedReason string = "ChartPullSucceeded" -) -``` - ## Spec examples -### Helm repository - Public Helm repository: ```yaml @@ -155,68 +97,15 @@ metadata: namespace: default type: Opaque data: - username: - password: - certFile: - keyFile: - caFile: -``` - -### Helm chart - -Pinned version: - -```yaml -apiVersion: source.fluxcd.io/v1alpha1 -kind: HelmChart -metadata: - name: redis - namespace: default - annotations: - # force sync trigger - source.fluxcd.io/syncAt: "2020-04-06T15:39:52+03:00" -spec: - name: redis - version: 10.5.7 - helmRepositoryRef: - name: stable -``` - -Semver range: - -```yaml -apiVersion: source.fluxcd.io/v1alpha1 -kind: HelmChart -metadata: - name: redis - namespace: default -spec: - name: redis - version: ^10.0.0 - helmRepositoryRef: - name: stable -``` - -Interval: - -```yaml -apiVersion: source.fluxcd.io/v1alpha1 -kind: HelmChart -metadata: - name: redis - namespace: default -spec: - name: redis - version: ^10.0.0 - helmRepositoryRef: - name: stable - interval: 30m + username: + password: + certFile: + keyFile: + caFile: ``` ## Status examples -### Helm repository - Successful indexation: ```yaml @@ -253,30 +142,3 @@ status: status: "False" type: Ready ``` - -### Helm chart - -Successful chart pull: - -```yaml -status: - url: http:///helmcharts/redis-default/redis-10.5.7.tgz - conditions: - - lastTransitionTime: "2020-04-10T09:34:45Z" - message: Fetched artifact are available at /data/helmcharts/redis-default/redis-10.5.7.tgz - reason: ChartPullSucceeded - status: "True" - type: Ready -``` - -Failed chart pull: - -```yaml -status: - conditions: - - lastTransitionTime: "2020-04-10T09:34:45Z" - message: '' - reason: ChartPullFailed - status: "False" - type: Ready -``` From e002278e35aa759d0e32ac67fcbd251d2dc35adc Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 14 Apr 2020 10:54:13 +0200 Subject: [PATCH 15/21] Improve comments --- docs/spec/v1alpha1/common.md | 6 +++++- docs/spec/v1alpha1/gitrepositories.md | 11 ++++++++--- docs/spec/v1alpha1/helmcharts.md | 4 ++-- docs/spec/v1alpha1/helmrepositories.md | 6 +++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 092bcec0..70f50011 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -109,11 +109,15 @@ const ( const ( // InitializingReason represents the fact that a given source is being initialized. InitializingReason string = "Initializing" + // URLInvalidReason represents the fact that a given source has an invalid URL. URLInvalidReason string = "URLInvalid" + // StorageOperationFailedReason signals a failure caused by a storage operation. StorageOperationFailedReason string = "StorageOperationFailed" - // AuthenticationFailedReason represents the fact that the provided credentials are not valid. + + // AuthenticationFailedReason represents the fact that a given secret does not + // have the required fields or the provided credentials do not match. AuthenticationFailedReason string = "AuthenticationFailed" ) ``` diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 945fe6e8..3fa9274c 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -9,13 +9,17 @@ an archive. Git repository: ```go -// GitRepositorySpec defines the desired state of GitRepository. +// GitRepositorySpec defines the desired state of a Git repository. type GitRepositorySpec struct { // The repository URL, can be a HTTP or SSH address. // +kubebuilder:validation:Pattern="^(http|https|ssh)://" URL string `json:"url"` // The secret name containing the Git credentials. + // For HTTPS repositories the secret must contain username and password + // fields. + // For SSH repositories the secret must contain identity, identity.pub and + // known_hosts fields. // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` @@ -47,7 +51,7 @@ type GitRepositoryRef struct { SemVer string `json:"semver"` // The git commit sha to checkout, if specified branch and tag filters will - // be ignored. + // ignored. // +optional Commit string `json:"commit"` } @@ -56,7 +60,7 @@ type GitRepositoryRef struct { ### Status ```go -// GitRepositoryStatus defines the observed state of GitRepository. +// GitRepositoryStatus defines the observed state of the GitRepository. type GitRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` @@ -79,6 +83,7 @@ const ( // GitOperationSucceedReason represents the fact that the git // clone, pull and checkout operations succeeded. GitOperationSucceedReason string = "GitOperationSucceed" + // GitOperationFailedReason represents the fact that the git // clone, pull or checkout operations failed. GitOperationFailedReason string = "GitOperationFailed" diff --git a/docs/spec/v1alpha1/helmcharts.md b/docs/spec/v1alpha1/helmcharts.md index 2a27101a..360a1253 100644 --- a/docs/spec/v1alpha1/helmcharts.md +++ b/docs/spec/v1alpha1/helmcharts.md @@ -25,8 +25,8 @@ type HelmChartSpec struct { // +required HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` - // The interval at which to check the Helm repository for updates. - // Defaults to the interval of the referenced HelmRepository. + // The interval at which to check the referenced HelmRepository index + // for updates. Defaults to the interval of the referenced HelmRepository. // +optional Interval metav1.Duration `json:"interval,omitempty"` } diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index 851a3ac1..1942129c 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -18,6 +18,9 @@ type HelmRepositorySpec struct { // The name of the secret containing authentication credentials // for the Helm repository. + // For HTTP/S basic auth the secret must contain username and password + // fields. + // For TLS the secret must contain caFile, keyFile and caCert fields. // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` @@ -30,7 +33,7 @@ type HelmRepositorySpec struct { ### Status ```go -// HelmRepositoryStatus defines the observed state of HelmRepository +// HelmRepositoryStatus defines the observed state of the HelmRepository. type HelmRepositoryStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` @@ -52,6 +55,7 @@ const ( // IndexationFailedReason represents the fact that the indexation // of the given Helm repository failed. IndexationFailedReason string = "IndexationFailed" + // IndexationSucceededReason represents the fact that the indexation // of the given Helm repository succeeded. IndexationSucceedReason string = "IndexationSucceed" From e0eb330d42a197917c5642a9b849de07db478cad Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 14 Apr 2020 11:07:06 +0200 Subject: [PATCH 16/21] Add source interface spec --- docs/spec/v1alpha1/common.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 70f50011..560e8250 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -4,6 +4,21 @@ Common defines resources used across all source types. ## Specification +### Source interface + +Source objects should adhere to the `Source` interface. This interface exposes the [interval](#source-synchronization) +and [artifact](#source-status) of the source to clients without the prerequisite of knowing the source kind: + +````go +type Source interface { + // GetInterval returns the interval at which the source is updated. + GetInterval() metav1.Duration + + // GetArtifact returns the latest artifact from the source, or nil. + GetArtifact() *Artifact +} +```` + ### Source synchronization Source objects should contain a `spec.interval` field that tells the controller at which interval to check for updates: From 51f82e72a8a5530ae7a7e04ef9c147cb7b6144c6 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 14 Apr 2020 11:16:29 +0200 Subject: [PATCH 17/21] Drop interval fallback for HelmChart --- docs/spec/v1alpha1/helmcharts.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/spec/v1alpha1/helmcharts.md b/docs/spec/v1alpha1/helmcharts.md index 360a1253..68fb0065 100644 --- a/docs/spec/v1alpha1/helmcharts.md +++ b/docs/spec/v1alpha1/helmcharts.md @@ -26,9 +26,9 @@ type HelmChartSpec struct { HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` // The interval at which to check the referenced HelmRepository index - // for updates. Defaults to the interval of the referenced HelmRepository. - // +optional - Interval metav1.Duration `json:"interval,omitempty"` + // for updates. + // +required + Interval metav1.Duration `json:"interval"` } ``` From 7323a45504426b556cae4bdc19c1e7ea0cab1a17 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Tue, 14 Apr 2020 19:09:18 +0300 Subject: [PATCH 18/21] Add signature verification to Git spec --- docs/spec/v1alpha1/common.md | 4 ++ docs/spec/v1alpha1/gitrepositories.md | 78 +++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index 560e8250..f41d65e7 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -134,6 +134,10 @@ const ( // AuthenticationFailedReason represents the fact that a given secret does not // have the required fields or the provided credentials do not match. AuthenticationFailedReason string = "AuthenticationFailed" + + // VerificationFailedReason represents the fact that the cryptographic provenance + // verification for the source failed. + VerificationFailedReason string = "VerificationFailed" ) ``` diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 3fa9274c..7addbce8 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -22,14 +22,18 @@ type GitRepositorySpec struct { // known_hosts fields. // +optional SecretRef *v1.LocalObjectReference `json:"secretRef,omitempty"` - + + // The interval at which to check for repository updates. + Interval metav1.Duration `json:"interval"` + // The git reference to checkout and monitor for changes, defaults to // master branch. // +optional Reference *GitRepositoryRef `json:"ref,omitempty"` - // The interval at which to check for repository updates. - Interval metav1.Duration `json:"interval"` + // Verify OpenPGP signature for the commit that HEAD points to. + // +optional + Verification *GitRepositoryVerification `json:"verify,omitempty"` } ``` @@ -57,6 +61,20 @@ type GitRepositoryRef struct { } ``` +Git repository cryptographic provenance verification: + +```go +// GitRepositoryVerification defines the OpenPGP signature verification process. +type GitRepositoryVerification struct { + // Mode describes what git object should be verified, currently ('head'). + // +kubebuilder:validation:Enum=head + Mode string `json:"mode"` + + // The secret name containing the public keys of all trusted git authors. + SecretRef corev1.LocalObjectReference `json:"secretRef"` +} +``` + ### Status ```go @@ -230,6 +248,46 @@ kubectl create secret generic ssh-credentials \ --from-file=./known_hosts ``` +Verify the OpenPGP signature for the commit that master branch HEAD points to: + +```yaml +apiVersion: source.fluxcd.io/v1alpha1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master + verify: + mode: head + secretRef: + name: pgp-public-keys +--- +apiVersion: v1 +kind: Secret +metadata: + name: pgp-public-keys + namespace: default +type: Opaque +data: + author1.asc: + author2.asc: +``` + +Example of generating the PGP public keys secret: + +```bash +gpg --export --armor 3CB12BA185C47B67 > author1.asc +gpg --export --armor 6A7436E8790F8689 > author2.asc + +kubectl create secret generic pgp-public-keys \ + --from-file=author1.asc \ + --from-file=author2.asc +``` + ## Status examples Successful sync: @@ -251,7 +309,7 @@ status: url: http:///gitrepository/podinfo-default/latest.tar.gz ``` -Failed sync: +Failed authentication: ```yaml status: @@ -264,6 +322,18 @@ status: type: Ready ``` +Failed PGP signature verification: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-04-06T06:48:59Z" + message: 'PGP signature of {Stefan Prodan 2020-04-04 13:36:58 +0300 +0300} can not be verified' + reason: VerificationFailed + status: "False" + type: Ready +``` + Wait for condition: ```bash From 0084e56971b54eaab3f999beb77fe47ed65c4e53 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 15 Apr 2020 12:50:16 +0300 Subject: [PATCH 19/21] Add impact to Flux - reword motivation by squaremo --- docs/spec/README.md | 67 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/docs/spec/README.md b/docs/spec/README.md index fe998f6c..aed1027e 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -1,44 +1,28 @@ # Source Controller Proposal -## Context - -The desired state of a cluster is made out of Kubernetes objects, these objects are expressed in `.yaml` format and -are applied on the cluster by operators running inside the cluster. An operator's role is to fetch the Kubernetes -objects, run transformations on them and reconcile the cluster state with the resulting manifest. - -For an operator to acquire the resources that make up the desired state it needs to understand the communication -protocol and the authentication scheme, verify the authenticity of a source and deal with rate limits and retries. -In the FluxCD organization there are currently two operators that perform such operations. Both Flux and -Helm Operator connect to Git repositories to fetch Kubernetes objects, they need to maintain an up-to-date mirror -of one or several repos. Besides Git, Helm Operator needs to connect to Helm repositories hosted on public or -private HTTPS servers. +The main goal is to define a set of Kubernetes objects that cluster +admins and various automated operators can interact with to offload +the sources (e.g. Git and Helm repositories) registration, authentication, +verification and resource fetching to a dedicated controller. ## Motivation -Each Flux or Helm Operator instance maintains its own Git repository mirror even if all of them -point to the same source. If the Git repository host becomes unavailable, the cluster state will diverge from the last -known desired state since the operators will stop the reconciliation due to pull errors. +Each Flux and each Helm operator mirrors the Git repositories they are +using, in the same way, using the same code. But other components +might benefit from access to the source mirrors, and Flux and the Helm +operator could work more in sympathy with Kubernetes by factoring it out. -Decoupling the Kubernetes objects acquisition from the reconciliation process with an in-cluster -source manager would make Flux and Helm Operator resilient to outbound connectivity issues and would -simplify the state machine(s) that these controllers operate. +If "sources" (usually git repos, but also Helm charts and potentially +other things) existed in their own right as Kubernetes resources, +components like Flux and Helm operator could use standard Kubernetes +mechanisms to build on them; and, they could be managed independently +of the components using them. -Managing the source operations in a dedicated controller could enable Flux to compose the desire state of a cluster -from multiple source. Furthermore, the manifests transformation process could be performed by 3rd party tools -(e.g. kustomize, jk, tanka, cue run by Tekton pipelines or Kubernetes Jobs) -that subscribe to source changes events. +## API Specification -The source controller could enable pinning the cluster desired state to a specific Git commit or Git tag. +* [v1alpha1](v1alpha1/README.md) -For teams that are using semantic versioning, the source controller could monitor the Git repository tags -and set the cluster desired state to the latest release or to a tag that matches a semver pattern. -In a similar fashion, a semver pattern could trigger Helm chart upgrades without manual intervention from users. - -## Goals - -The main goal is to define a set of Kubernetes objects that cluster admins and various automated operators -can interact with to offload the sources (e.g. Git and Helm repositories) -registration, authentication and resource fetching to a dedicated controller. +## Implementation The controller implementation will watch for source objects in a cluster and act on them. The actions performed by the source controller could be: @@ -46,12 +30,25 @@ The actions performed by the source controller could be: * authenticate to sources and validate authenticity * detect source changes based on update policies (semver) * fetch resources on-demand and on-a-schedule -* package the fetched resources into a well known format (tar.gz) +* package the fetched resources into a well known format (tar.gz, yaml) * store the artifacts locally * make the artifacts addressable by their source identifier (sha, version, ts) * make the artifacts available in-cluster to interested 3rd parties * notify interested 3rd parties of source changes and availability (status conditions, events, hooks) -## API Specification +## Impact to Flux -* [v1alpha1](v1alpha1/README.md) +Having a dedicated controller that manages Git repositories defined with Kubernetes custom resources would: +* simplify Flux configuration as fluxd could subscribe to Git sources in-cluster and pull the artifacts +automatically without manual intervention from users to reconfigure and redeploy FLux +* improve the installation experience as users will not have to patch fluxd's deployment to inject +the HTTPS basic auth credentials, change the source URL or other Git and PGP related settings +* enable fluxd to compose the desired state of a cluster from multiple sources by applying all artifacts present in flux namespace +* enable fluxd to apply manifests coming from other sources than Git, e.g. S3 buckets +* allow fluxd to run under a non-root user as it wouldn't need to shell out to ssh-keygen, git or pgp +* enable fluxd to apply manifests coming from the most recent semver tag of a Git repository +* allow user to pin the cluster desired state to a specific Git commit or Git tag + +## Impact to Helm Operator + +TODO \ No newline at end of file From 8b4432fc5ce252e4ee7b980f19c09a14ee5edebc Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 15 Apr 2020 12:33:12 +0200 Subject: [PATCH 20/21] Add impact to Helm Operator --- docs/spec/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/spec/README.md b/docs/spec/README.md index aed1027e..60c1c946 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -26,6 +26,7 @@ of the components using them. The controller implementation will watch for source objects in a cluster and act on them. The actions performed by the source controller could be: + * validate source definitions * authenticate to sources and validate authenticity * detect source changes based on update policies (semver) @@ -39,6 +40,7 @@ The actions performed by the source controller could be: ## Impact to Flux Having a dedicated controller that manages Git repositories defined with Kubernetes custom resources would: + * simplify Flux configuration as fluxd could subscribe to Git sources in-cluster and pull the artifacts automatically without manual intervention from users to reconfigure and redeploy FLux * improve the installation experience as users will not have to patch fluxd's deployment to inject @@ -51,4 +53,13 @@ the HTTPS basic auth credentials, change the source URL or other Git and PGP rel ## Impact to Helm Operator -TODO \ No newline at end of file +Having a dedicated controller that manages Helm repositories and charts defined with Kubernetes custom +resources would: + +* simplify the Helm Operator configuration as repository and chart definitions can be re-used across + `HelmRelease` resources (see [fluxcd/helm-operator#142](https://github.com/fluxcd/helm-operator/issues/142)) +* improve the user experience as repositories requiring authentication will no longer require a + `repositories.yaml` import / file mount +* simplify the architecture of the Helm Operator as it allows the operator to work with a single + source type (`HelmChart`) and way of preparing and executing installations and/or upgrades +* allow the Helm Operator to run under a non-root user as it wouldn't need to shell out to git From 000bf41dc2f2034c1b94150bb5d164600ceed734 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 16 Apr 2020 12:08:09 +0200 Subject: [PATCH 21/21] Address comments --- docs/spec/v1alpha1/gitrepositories.md | 5 +++-- docs/spec/v1alpha1/helmcharts.md | 9 +++++---- docs/spec/v1alpha1/helmrepositories.md | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/spec/v1alpha1/gitrepositories.md b/docs/spec/v1alpha1/gitrepositories.md index 7addbce8..f168db3f 100644 --- a/docs/spec/v1alpha1/gitrepositories.md +++ b/docs/spec/v1alpha1/gitrepositories.md @@ -1,6 +1,6 @@ # Git Repositories -The `GitReposiory` API defines a source for artifacts coming from Git. The +The `GitRepository` API defines a source for artifacts coming from Git. The resource exposes the latest synchronized state from Git as an artifact in an archive. @@ -9,7 +9,8 @@ an archive. Git repository: ```go -// GitRepositorySpec defines the desired state of a Git repository. +// GitRepositorySpec gives the specification for fetching a Git repository as +// a source. type GitRepositorySpec struct { // The repository URL, can be a HTTP or SSH address. // +kubebuilder:validation:Pattern="^(http|https|ssh)://" diff --git a/docs/spec/v1alpha1/helmcharts.md b/docs/spec/v1alpha1/helmcharts.md index 68fb0065..a5d90476 100644 --- a/docs/spec/v1alpha1/helmcharts.md +++ b/docs/spec/v1alpha1/helmcharts.md @@ -9,7 +9,7 @@ exposes the latest pulled chart for the defined version as an artifact. Helm chart: ```go -// HelmChart defines the desired state of a Helm chart. +// HelmChartSpec defines the desired state of a Helm chart source. type HelmChartSpec struct { // The name of the Helm chart, as made available by the referenced // Helm repository. @@ -36,7 +36,7 @@ type HelmChartSpec struct { ```go // HelmChartStatus defines the observed state of the HelmChart. -type HelmRepositoryStatus struct { +type HelmChartStatus struct { // +optional Conditions []SourceCondition `json:"conditions,omitempty"` @@ -56,10 +56,11 @@ type HelmRepositoryStatus struct { const ( // ChartPullFailedReason represents the fact that the pull of the // given Helm chart failed. - ChartPullFailedReason string = "ChartPullFailed" + ChartPullFailedReason string = "ChartPullFailed" + // ChartPullSucceededReason represents the fact that the pull of // the given Helm chart succeeded. - ChartPullSucceedReason string = "ChartPullSucceeded" + ChartPullSucceededReason string = "ChartPullSucceeded" ) ``` diff --git a/docs/spec/v1alpha1/helmrepositories.md b/docs/spec/v1alpha1/helmrepositories.md index 1942129c..0649a1f8 100644 --- a/docs/spec/v1alpha1/helmrepositories.md +++ b/docs/spec/v1alpha1/helmrepositories.md @@ -9,7 +9,7 @@ an artifact. Helm repository: ```go -// HelmRepository defines the reference to a Helm repository. +// HelmRepositorySpec defines the reference to a Helm repository. type HelmRepositorySpec struct { // The Helm repository URL, a valid URL contains at least a // protocol and host.