diff --git a/api/v1beta2/gitrepository_types.go b/api/v1beta2/gitrepository_types.go index 07091280..e02cf005 100644 --- a/api/v1beta2/gitrepository_types.go +++ b/api/v1beta2/gitrepository_types.go @@ -29,85 +29,116 @@ const ( // GitRepositoryKind is the string representation of a GitRepository. GitRepositoryKind = "GitRepository" - // GoGitImplementation represents the go-git Git implementation kind. + // GoGitImplementation for performing Git operations using go-git. GoGitImplementation = "go-git" - // LibGit2Implementation represents the git2go Git implementation kind. + // LibGit2Implementation for performing Git operations using libgit2. LibGit2Implementation = "libgit2" ) const ( - // IncludeUnavailableCondition indicates one of the includes is not available. For example, because it does not - // exist, or does not have an Artifact. - // This is a "negative polarity" or "abnormal-true" type, and is only present on the resource if it is True. + // IncludeUnavailableCondition indicates one of the includes is not + // available. For example, because it does not exist, or does not have an + // Artifact. + // This is a "negative polarity" or "abnormal-true" type, and is only + // present on the resource if it is True. IncludeUnavailableCondition string = "IncludeUnavailable" ) -// GitRepositorySpec defines the desired state of a Git repository. +// GitRepositorySpec specifies the required configuration to produce an +// Artifact for a Git repository. type GitRepositorySpec struct { - // The repository URL, can be a HTTP/S or SSH address. + // URL specifies the Git repository URL, can be an HTTP/S or SSH address. // +kubebuilder:validation:Pattern="^(http|https|ssh)://" // +required URL string `json:"url"` - // The secret name containing the Git credentials. - // For HTTPS repositories the secret must contain username and password fields. - // For SSH repositories the secret must contain 'identity', 'identity.pub' and 'known_hosts' fields. + // SecretRef specifies the Secret containing authentication credentials for + // the GitRepository. + // For HTTPS repositories the Secret must contain 'username' and 'password' + // fields. + // For SSH repositories the Secret must contain 'identity', 'identity.pub' + // and 'known_hosts' fields. // +optional SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"` - // The interval at which to check for repository updates. + // Interval at which to check the GitRepository for updates. // +required Interval metav1.Duration `json:"interval"` - // The timeout for remote Git operations like cloning, defaults to 60s. + // Timeout for Git operations like cloning, defaults to 60s. // +kubebuilder:default="60s" // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` - // The Git reference to checkout and monitor for changes, defaults to - // master branch. + // Reference specifies the Git reference to resolve and monitor for + // changes, defaults to the 'master' branch. // +optional Reference *GitRepositoryRef `json:"ref,omitempty"` - // Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to. + // Verification specifies the configuration to verify the Git commit + // signature(s). // +optional Verification *GitRepositoryVerification `json:"verify,omitempty"` - // Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). - // If not provided, a default will be used, consult the documentation for your version to find out what those are. + // Ignore overrides the set of excluded patterns in the .sourceignore format + // (which is the same as .gitignore). If not provided, a default will be used, + // consult the documentation for your version to find out what those are. // +optional Ignore *string `json:"ignore,omitempty"` - // Suspend tells the controller to suspend the reconciliation of this source. - // This flag tells the controller to suspend the reconciliation of this source. + // Suspend tells the controller to suspend the reconciliation of this + // GitRepository. // +optional Suspend bool `json:"suspend,omitempty"` - // Determines which git client library to use. - // Defaults to go-git, valid values are ('go-git', 'libgit2'). + // GitImplementation specifies which Git client library implementation to + // use. Defaults to 'go-git', valid values are ('go-git', 'libgit2'). // +kubebuilder:validation:Enum=go-git;libgit2 // +kubebuilder:default:=go-git // +optional GitImplementation string `json:"gitImplementation,omitempty"` - // When enabled, after the clone is created, initializes all submodules within, using their default settings. + // RecurseSubmodules enables the initialization of all submodules within + // the GitRepository as cloned from the URL, using their default settings. // This option is available only when using the 'go-git' GitImplementation. // +optional RecurseSubmodules bool `json:"recurseSubmodules,omitempty"` - // Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for - // this resource. + // Include specifies a list of GitRepository resources which Artifacts + // should be included in the Artifact produced for this GitRepository. Include []GitRepositoryInclude `json:"include,omitempty"` - // AccessFrom defines an Access Control List for allowing cross-namespace references to this object. + // AccessFrom specifies an Access Control List for allowing cross-namespace + // references to this object. // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` } +// GitRepositoryInclude specifies a reference to a GitRepository which Artifact +// (sub-)contents must be included, and where they should be placed. +type GitRepositoryInclude struct { + // GitRepositoryRef specifies the GitRepository which Artifact contents + // must be included. + GitRepositoryRef meta.LocalObjectReference `json:"repository"` + + // FromPath specifies the path to copy contents from, defaults to the root + // of the Artifact. + // +optional + FromPath string `json:"fromPath"` + + // ToPath specifies the path to copy contents to, defaults to the name of + // the GitRepositoryRef. + // +optional + ToPath string `json:"toPath"` +} + +// GetFromPath returns the specified FromPath. func (in *GitRepositoryInclude) GetFromPath() string { return in.FromPath } +// GetToPath returns the specified ToPath, falling back to the name of the +// GitRepositoryRef. func (in *GitRepositoryInclude) GetToPath() string { if in.ToPath == "" { return in.GitRepositoryRef.Name @@ -115,52 +146,48 @@ func (in *GitRepositoryInclude) GetToPath() string { return in.ToPath } -// GitRepositoryInclude defines a source with a from and to path. -type GitRepositoryInclude struct { - // Reference to a GitRepository to include. - GitRepositoryRef meta.LocalObjectReference `json:"repository"` - - // The path to copy contents from, defaults to the root directory. - // +optional - FromPath string `json:"fromPath"` - - // The path to copy contents to, defaults to the name of the source ref. - // +optional - ToPath string `json:"toPath"` -} - -// GitRepositoryRef defines the Git ref used for pull and checkout operations. +// GitRepositoryRef specifies the Git reference to resolve and checkout. type GitRepositoryRef struct { - // The Git branch to checkout, defaults to master. + // Branch to check out, defaults to 'master' if no other field is defined. + // + // When GitRepositorySpec.GitImplementation is set to 'go-git', a shallow + // clone of the specified branch is performed. // +optional Branch string `json:"branch,omitempty"` - // The Git tag to checkout, takes precedence over Branch. + // Tag to check out, takes precedence over Branch. // +optional Tag string `json:"tag,omitempty"` - // The Git tag semver expression, takes precedence over Tag. + // SemVer tag expression to check out, takes precedence over Tag. // +optional SemVer string `json:"semver,omitempty"` - // The Git commit SHA to checkout, if specified Tag filters will be ignored. + // Commit SHA to check out, takes precedence over all reference fields. + // + // When GitRepositorySpec.GitImplementation is set to 'go-git', this can be + // combined with Branch to shallow clone the branch, in which the commit is + // expected to exist. // +optional Commit string `json:"commit,omitempty"` } -// GitRepositoryVerification defines the OpenPGP signature verification process. +// GitRepositoryVerification specifies the Git commit signature verification +// strategy. type GitRepositoryVerification struct { - // Mode describes what Git object should be verified, currently ('head'). + // Mode specifies what Git object should be verified, currently ('head'). // +kubebuilder:validation:Enum=head Mode string `json:"mode"` - // SecretRef containing the public keys of all trusted Git authors. + // SecretRef specifies the Secret containing the public keys of trusted Git + // authors. SecretRef meta.LocalObjectReference `json:"secretRef,omitempty"` } -// GitRepositoryStatus defines the observed state of a Git repository. +// GitRepositoryStatus records the observed state of a Git repository. type GitRepositoryStatus struct { - // ObservedGeneration is the last observed generation. + // ObservedGeneration is the last observed generation of the GitRepository + // object. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -168,15 +195,18 @@ type GitRepositoryStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // URL is the fetch link for the artifact output of the last repository sync. + // URL is the dynamic fetch link for the latest Artifact. + // It is provided on a "best effort" basis, and using the precise + // GitRepositoryStatus.Artifact data is recommended. // +optional URL string `json:"url,omitempty"` - // Artifact represents the output of the last successful repository sync. + // Artifact represents the last successful GitRepository reconciliation. // +optional Artifact *Artifact `json:"artifact,omitempty"` - // IncludedArtifacts represents the included artifacts from the last successful repository sync. + // IncludedArtifacts contains a list of the last successfully included + // Artifacts as instructed by GitRepositorySpec.Include. // +optional IncludedArtifacts []*Artifact `json:"includedArtifacts,omitempty"` @@ -184,10 +214,12 @@ type GitRepositoryStatus struct { } const ( - // GitOperationSucceedReason represents the fact that the git clone, pull and checkout operations succeeded. - GitOperationSucceedReason string = "GitOperationSucceed" + // GitOperationSucceedReason signals that a Git operation (e.g. clone, + // checkout, etc.) succeeded. + GitOperationSucceedReason string = "GitOperationSucceeded" - // GitOperationFailedReason represents the fact that the git clone, pull or checkout operations failed. + // GitOperationFailedReason signals that a Git operation (e.g. clone, + // checkout, etc.) failed. GitOperationFailedReason string = "GitOperationFailed" ) @@ -201,28 +233,18 @@ func (in *GitRepository) SetConditions(conditions []metav1.Condition) { in.Status.Conditions = conditions } -// GetRequeueAfter returns the duration after which the source must be reconciled again. +// GetRequeueAfter returns the duration after which the GitRepository must be +// reconciled again. func (in GitRepository) GetRequeueAfter() time.Duration { return in.Spec.Interval.Duration } -// GetInterval returns the interval at which the source is reconciled. -// Deprecated: use GetRequeueAfter instead. -func (in GitRepository) GetInterval() metav1.Duration { - return in.Spec.Interval -} - -// GetArtifact returns the latest artifact from the source if present in the status sub-resource. +// GetArtifact returns the latest Artifact from the GitRepository if present in +// the status sub-resource. func (in *GitRepository) GetArtifact() *Artifact { return in.Status.Artifact } -// GetStatusConditions returns a pointer to the Status.Conditions slice. -// Deprecated: use GetConditions instead. -func (in *GitRepository) GetStatusConditions() *[]metav1.Condition { - return &in.Status.Conditions -} - // +genclient // +genclient:Namespaced // +kubebuilder:storageversion @@ -234,7 +256,7 @@ func (in *GitRepository) GetStatusConditions() *[]metav1.Condition { // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="" -// GitRepository is the Schema for the gitrepositories API +// GitRepository is the Schema for the gitrepositories API. type GitRepository struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -244,7 +266,7 @@ type GitRepository struct { Status GitRepositoryStatus `json:"status,omitempty"` } -// GitRepositoryList contains a list of GitRepository +// GitRepositoryList contains a list of GitRepository objects. // +kubebuilder:object:root=true type GitRepositoryList struct { metav1.TypeMeta `json:",inline"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 2c0291fb..334c1def 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -365,7 +365,7 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: GitRepository is the Schema for the gitrepositories API + description: GitRepository is the Schema for the gitrepositories API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -380,10 +380,11 @@ spec: metadata: type: object spec: - description: GitRepositorySpec defines the desired state of a Git repository. + description: GitRepositorySpec specifies the required configuration to + produce an Artifact for a Git repository. properties: accessFrom: - description: AccessFrom defines an Access Control List for allowing + description: AccessFrom specifies an Access Control List for allowing cross-namespace references to this object. properties: namespaceSelectors: @@ -411,8 +412,9 @@ spec: type: object gitImplementation: default: go-git - description: Determines which git client library to use. Defaults - to go-git, valid values are ('go-git', 'libgit2'). + description: GitImplementation specifies which Git client library + implementation to use. Defaults to 'go-git', valid values are ('go-git', + 'libgit2'). enum: - go-git - libgit2 @@ -424,18 +426,20 @@ spec: to find out what those are. type: string include: - description: Include defines a list of GitRepository resources which - artifacts should be included in the artifact produced for this resource. + description: Include specifies a list of GitRepository resources which + Artifacts should be included in the Artifact produced for this GitRepository. items: - description: GitRepositoryInclude defines a source with a from and - to path. + description: GitRepositoryInclude specifies a reference to a GitRepository + which Artifact (sub-)contents must be included, and where they + should be placed. properties: fromPath: - description: The path to copy contents from, defaults to the - root directory. + description: FromPath specifies the path to copy contents from, + defaults to the root of the Artifact. type: string repository: - description: Reference to a GitRepository to include. + description: GitRepositoryRef specifies the GitRepository which + Artifact contents must be included. properties: name: description: Name of the referent. @@ -444,45 +448,52 @@ spec: - name type: object toPath: - description: The path to copy contents to, defaults to the name - of the source ref. + description: ToPath specifies the path to copy contents to, + defaults to the name of the GitRepositoryRef. type: string required: - repository type: object type: array interval: - description: The interval at which to check for repository updates. + description: Interval at which to check the GitRepository for updates. type: string recurseSubmodules: - description: When enabled, after the clone is created, initializes - all submodules within, using their default settings. This option - is available only when using the 'go-git' GitImplementation. + description: RecurseSubmodules enables the initialization of all submodules + within the GitRepository as cloned from the URL, using their default + settings. This option is available only when using the 'go-git' + GitImplementation. type: boolean ref: - description: The Git reference to checkout and monitor for changes, - defaults to master branch. + description: Reference specifies the Git reference to resolve and + monitor for changes, defaults to the 'master' branch. properties: branch: - description: The Git branch to checkout, defaults to master. + description: "Branch to check out, defaults to 'master' if no + other field is defined. \n When GitRepositorySpec.GitImplementation + is set to 'go-git', a shallow clone of the specified branch + is performed." type: string commit: - description: The Git commit SHA to checkout, if specified Tag - filters will be ignored. + description: "Commit SHA to check out, takes precedence over all + reference fields. \n When GitRepositorySpec.GitImplementation + is set to 'go-git', this can be combined with Branch to shallow + clone the branch, in which the commit is expected to exist." type: string semver: - description: The Git tag semver expression, takes precedence over - Tag. + description: SemVer tag expression to check out, takes precedence + over Tag. type: string tag: - description: The Git tag to checkout, takes precedence over Branch. + description: Tag to check out, takes precedence over Branch. type: string type: object secretRef: - description: The secret name containing the Git credentials. For HTTPS - repositories the secret must contain username and password fields. - For SSH repositories the secret must contain 'identity', 'identity.pub' - and 'known_hosts' fields. + description: SecretRef specifies the Secret containing authentication + credentials for the GitRepository. For HTTPS repositories the Secret + must contain 'username' and 'password' fields. For SSH repositories + the Secret must contain 'identity', 'identity.pub' and 'known_hosts' + fields. properties: name: description: Name of the referent. @@ -492,31 +503,31 @@ spec: type: object suspend: description: Suspend tells the controller to suspend the reconciliation - of this source. This flag tells the controller to suspend the reconciliation - of this source. + of this GitRepository. type: boolean timeout: default: 60s - description: The timeout for remote Git operations like cloning, defaults - to 60s. + description: Timeout for Git operations like cloning, defaults to + 60s. type: string url: - description: The repository URL, can be a HTTP/S or SSH address. + description: URL specifies the Git repository URL, can be an HTTP/S + or SSH address. pattern: ^(http|https|ssh):// type: string verify: - description: Verification defines the configuration to verify the - OpenPGP signature for the Git commit HEAD points to. + description: Verification specifies the configuration to verify the + Git commit signature(s). properties: mode: - description: Mode describes what Git object should be verified, + description: Mode specifies what Git object should be verified, currently ('head'). enum: - head type: string secretRef: - description: SecretRef containing the public keys of all trusted - Git authors. + description: SecretRef specifies the Secret containing the public + keys of trusted Git authors. properties: name: description: Name of the referent. @@ -534,11 +545,11 @@ spec: status: default: observedGeneration: -1 - description: GitRepositoryStatus defines the observed state of a Git repository. + description: GitRepositoryStatus records the observed state of a Git repository. properties: artifact: - description: Artifact represents the output of the last successful - repository sync. + description: Artifact represents the last successful GitRepository + reconciliation. properties: checksum: description: Checksum is the SHA256 checksum of the Artifact file. @@ -639,8 +650,8 @@ spec: type: object type: array includedArtifacts: - description: IncludedArtifacts represents the included artifacts from - the last successful repository sync. + description: IncludedArtifacts contains a list of the last successfully + included Artifacts as instructed by GitRepositorySpec.Include. items: description: Artifact represents the output of a Source reconciliation. properties: @@ -681,12 +692,14 @@ spec: be detected. type: string observedGeneration: - description: ObservedGeneration is the last observed generation. + description: ObservedGeneration is the last observed generation of + the GitRepository object. format: int64 type: integer url: - description: URL is the fetch link for the artifact output of the - last repository sync. + description: URL is the dynamic fetch link for the latest Artifact. + It is provided on a "best effort" basis, and using the precise GitRepositoryStatus.Artifact + data is recommended. type: string type: object type: object diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index dd67041d..4f7606ef 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -53,9 +53,9 @@ import ( "github.com/fluxcd/source-controller/pkg/sourceignore" ) -// gitRepoReadyConditions contains all the conditions information needed -// for GitRepository Ready status conditions summary calculation. -var gitRepoReadyConditions = summarize.Conditions{ +// gitRepositoryReadyCondition contains the information required to summarize a +// v1beta2.GitRepository Ready Condition. +var gitRepositoryReadyCondition = summarize.Conditions{ Target: meta.ReadyCondition, Owned: []string{ sourcev1.SourceVerifiedCondition, @@ -88,7 +88,7 @@ var gitRepoReadyConditions = summarize.Conditions{ // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories/finalizers,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch -// GitRepositoryReconciler reconciles a GitRepository object +// GitRepositoryReconciler reconciles a v1beta2.GitRepository object. type GitRepositoryReconciler struct { client.Client kuberecorder.EventRecorder @@ -104,9 +104,9 @@ type GitRepositoryReconcilerOptions struct { DependencyRequeueInterval time.Duration } -// gitRepoReconcilerFunc is the function type for all the Git repository -// reconciler functions. -type gitRepoReconcilerFunc func(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) +// gitRepositoryReconcileFunc is the function type for all the +// v1beta2.GitRepository (sub)reconcile functions. +type gitRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { return r.SetupWithManagerAndOptions(mgr, GitRepositoryReconcilerOptions{}) @@ -156,7 +156,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques defer func() { summarizeHelper := summarize.NewHelper(r.EventRecorder, patchHelper) summarizeOpts := []summarize.Option{ - summarize.WithConditions(gitRepoReadyConditions), + summarize.WithConditions(gitRepositoryReadyCondition), summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), @@ -164,7 +164,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetInterval().Duration}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) @@ -188,7 +188,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Reconcile actual object - reconcilers := []gitRepoReconcilerFunc{ + reconcilers := []gitRepositoryReconcileFunc{ r.reconcileStorage, r.reconcileSource, r.reconcileInclude, @@ -198,17 +198,15 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques return } -// reconcile steps iterates through the actual reconciliation tasks for objec, -// it returns early on the first step that returns ResultRequeue or produces an -// error. -func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository, reconcilers []gitRepoReconcilerFunc) (sreconcile.Result, error) { +// reconcile iterates through the gitRepositoryReconcileFunc tasks for the +// object. It returns early on the first call that returns +// reconcile.ResultRequeue, or produces an error. +func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository, reconcilers []gitRepositoryReconcileFunc) (sreconcile.Result, error) { + // Mark as reconciling if generation differs if obj.Generation != obj.Status.ObservedGeneration { conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation) } - var artifact sourcev1.Artifact - var includes artifactSet - // Create temp dir for Git clone tmpDir, err := util.TempDirForObj("", obj) if err != nil { @@ -217,11 +215,20 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.G Reason: sourcev1.StorageOperationFailedReason, } } - defer os.RemoveAll(tmpDir) + defer func() { + if err = os.RemoveAll(tmpDir); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary working directory") + } + }() // Run the sub-reconcilers and build the result of reconciliation. - var res sreconcile.Result - var resErr error + var ( + artifact sourcev1.Artifact + includes artifactSet + + res sreconcile.Result + resErr error + ) for _, rec := range reconcilers { recResult, err := rec(ctx, obj, &artifact, &includes, tmpDir) // Exit immediately on ResultRequeue. @@ -241,13 +248,18 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.G return res, resErr } -// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// reconcileStorage ensures the current state of the storage matches the +// desired and previously observed state. // -// All artifacts for the resource except for the current one are garbage collected from the storage. -// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. -// If the object does not have an artifact in its Status object, a v1beta1.ArtifactUnavailableCondition is set. -// If the hostname of any of the URLs on the object do not match the current storage server hostname, they are updated. -func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) { +// All Artifacts for the object except for the current one in the Status are +// garbage collected from the Storage. +// If the Artifact in the Status of the object disappeared from the Storage, +// it is removed from the object. +// If the object does not have an Artifact in its Status, a Reconciling +// condition is added. +// The hostname of any URL in the Status of the object are updated, to ensure +// they match the Storage server hostname of current runtime. +func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.GitRepository, _ *sourcev1.Artifact, _ *artifactSet, _ string) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -271,19 +283,25 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sou return sreconcile.ResultSuccess, nil } -// reconcileSource ensures the upstream Git repository can be reached and checked out using the declared configuration, -// and observes its state. +// reconcileSource ensures the upstream Git repository and reference can be +// cloned and checked out using the specified configuration, and observes its +// state. // -// The repository is checked out to the given dir using the defined configuration, and in case of an error during the -// checkout process (including transient errors), it records v1beta1.FetchFailedCondition=True and returns early. -// On a successful checkout it removes v1beta1.FetchFailedCondition, and compares the current revision of HEAD to the -// artifact on the object, and records v1beta1.ArtifactOutdatedCondition if they differ. -// If instructed, the signature of the commit is verified if and recorded as v1beta1.SourceVerifiedCondition. If the -// signature can not be verified or the verification fails, the Condition=False and it returns early. -// If both the checkout and signature verification are successful, the given artifact pointer is set to a new artifact -// with the available metadata. -func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, - obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, _ *artifactSet, dir string) (sreconcile.Result, error) { +// The repository is cloned to the given dir, using the specified configuration +// to check out the reference. In case of an error during this process +// (including transient errors), it records v1beta2.FetchFailedCondition=True +// and returns early. +// On a successful checkout, it removes v1beta2.FetchFailedCondition and +// compares the current revision of HEAD to the revision of the Artifact in the +// Status of the object. It records v1beta2.ArtifactOutdatedCondition=True when +// they differ. +// If specified, the signature of the Git commit is verified. If the signature +// can not be verified or the verification fails, it records +// v1beta2.SourceVerifiedCondition=False and returns early. When successful, +// it records v1beta2.SourceVerifiedCondition=True. +// When all the above is successful, the given Artifact pointer is set to a new +// Artifact composed out of the gathered Git metadata. +func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, _ *artifactSet, dir string) (sreconcile.Result, error) { // Configure authentication strategy to access the source var authOpts *git.AuthOptions var err error @@ -374,15 +392,17 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, return sreconcile.ResultSuccess, nil } -// reconcileArtifact archives a new artifact to the storage, if the current observation on the object does not match the -// given data. +// reconcileArtifact archives a new Artifact to the Storage, if the current +// (Status) data on the object does not match the given. // -// The inspection of the given data to the object is differed, ensuring any stale observations as -// v1beta1.ArtifactUnavailableCondition and v1beta1.ArtifactOutdatedCondition are always deleted. -// If the given artifact and/or includes do not differ from the object's current, it returns early. -// Source ignore patterns are loaded, and the given directory is archived. -// On a successful archive, the artifact and includes in the status of the given object are set, and the symlink in the -// storage is updated to its path. +// The inspection of the given data to the object is differed, ensuring any +// stale observations like v1beta2.ArtifactOutdatedCondition are removed. +// If the given Artifact and/or artifactSet (includes) do not differ from the +// object's current, it returns early. +// Source ignore patterns are loaded, and the given directory is archived while +// taking these patterns into account. +// On a successful archive, the Artifact and Includes in the Status of the +// object are set, and the symlink in the Storage is updated to its path. func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) { // Always restore the Ready condition in case it got removed due to a transient error defer func() { @@ -475,13 +495,19 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *so return sreconcile.ResultSuccess, nil } -// reconcileInclude reconciles the declared includes from the object by copying their artifact (sub)contents to the -// declared paths in the given directory. +// reconcileInclude reconciles the on the object specified +// v1beta2.GitRepositoryInclude list by copying their Artifact (sub)contents to +// the specified paths in the given directory. // -// If an include is unavailable, it marks the object with v1beta1.IncludeUnavailableCondition and returns early. -// If the copy operations are successful, it deletes the v1beta1.IncludeUnavailableCondition from the object. -// If the artifactSet differs from the current set, it marks the object with v1beta1.ArtifactOutdatedCondition. -func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sourcev1.GitRepository, _ *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) { +// When one of the includes is unavailable, it marks the object with +// v1beta2.IncludeUnavailableCondition=True and returns early. +// When the copy operations are successful, it removes the +// v1beta2.IncludeUnavailableCondition from the object. +// When the composed artifactSet differs from the current set in the Status of +// the object, it marks the object with v1beta2.ArtifactOutdatedCondition=True. +func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sourcev1.GitRepository, + _ *sourcev1.Artifact, includes *artifactSet, dir string) (sreconcile.Result, error) { + artifacts := make(artifactSet, len(obj.Spec.Include)) for i, incl := range obj.Spec.Include { // Do this first as it is much cheaper than copy operations @@ -542,25 +568,16 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sou return sreconcile.ResultSuccess, nil } -// reconcileDelete handles the delete of an object. It first garbage collects all artifacts for the object from the -// artifact storage, if successful, the finalizer is removed from the object. -func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (sreconcile.Result, error) { - // Garbage collect the resource's artifacts - if err := r.garbageCollect(ctx, obj); err != nil { - // Return the error so we retry the failed garbage collection - return sreconcile.ResultEmpty, err - } - - // Remove our finalizer from the list - controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer) - - // Stop reconciliation as the object is being deleted - return sreconcile.ResultEmpty, nil -} - -// verifyCommitSignature verifies the signature of the given commit if a verification mode is configured on the object. +// verifyCommitSignature verifies the signature of the given Git commit, if a +// verification mode is specified on the object. +// If the signature can not be verified or the verification fails, it records +// v1beta2.SourceVerifiedCondition=False and returns. +// When successful, it records v1beta2.SourceVerifiedCondition=True. +// If no verification mode is specified on the object, the +// v1beta2.SourceVerifiedCondition Condition is removed. func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj *sourcev1.GitRepository, commit git.Commit) (sreconcile.Result, error) { - // Check if there is a commit verification is configured and remove any old observations if there is none + // Check if there is a commit verification is configured and remove any old + // observations if there is none if obj.Spec.Verification == nil || obj.Spec.Verification.Mode == "" { conditions.Delete(obj, sourcev1.SourceVerifiedCondition) return sreconcile.ResultSuccess, nil @@ -603,9 +620,28 @@ func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj return sreconcile.ResultSuccess, nil } -// garbageCollect performs a garbage collection for the given v1beta1.GitRepository. It removes all but the current -// artifact except for when the deletion timestamp is set, which will result in the removal of all artifacts for the -// resource. +// reconcileDelete handles the deletion of the object. +// It first garbage collects all Artifacts for the object from the Storage. +// Removing the finalizer from the object if successful. +func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (sreconcile.Result, error) { + // Garbage collect the resource's artifacts + if err := r.garbageCollect(ctx, obj); err != nil { + // Return the error so we retry the failed garbage collection + return sreconcile.ResultEmpty, err + } + + // Remove our finalizer from the list + controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer) + + // Stop reconciliation as the object is being deleted + return sreconcile.ResultEmpty, nil +} + +// garbageCollect performs a garbage collection for the given object. +// +// It removes all but the current Artifact from the Storage, unless the +// deletion timestamp on the object is set. Which will result in the +// removal of all Artifacts for the objects. func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourcev1.GitRepository) error { if !obj.DeletionTimestamp.IsZero() { if deleted, err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { @@ -633,9 +669,11 @@ func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourc return nil } -// eventLog records event and logs at the same time. This log is different from -// the debug log in the event recorder in the sense that this is a simple log, -// the event recorder debug log contains complete details about the event. +// eventLogf records event and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details +// about the event. func (r *GitRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { msg := fmt.Sprintf(messageFmt, args...) // Log and emit event. diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go index 6549647e..db0a0fc6 100644 --- a/controllers/gitrepository_controller_test.go +++ b/controllers/gitrepository_controller_test.go @@ -191,7 +191,7 @@ func TestGitRepositoryReconciler_Reconcile(t *testing.T) { }, timeout).Should(BeTrue()) // Check if the object status is valid. - condns := &status.Conditions{NegativePolarity: gitRepoReadyConditions.NegativePolarity} + condns := &status.Conditions{NegativePolarity: gitRepositoryReadyCondition.NegativePolarity} checker := status.NewChecker(testEnv.Client, testEnv.GetScheme(), condns) checker.CheckErr(ctx, obj) diff --git a/docs/api/source.md b/docs/api/source.md index 6f46cff8..43b51e56 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -240,7 +240,7 @@ BucketStatus

GitRepository

-

GitRepository is the Schema for the gitrepositories API

+

GitRepository is the Schema for the gitrepositories API.

@@ -303,7 +303,7 @@ string @@ -317,9 +317,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -332,7 +335,7 @@ Kubernetes meta/v1.Duration @@ -346,7 +349,7 @@ Kubernetes meta/v1.Duration @@ -360,8 +363,8 @@ GitRepositoryRef @@ -375,7 +378,8 @@ GitRepositoryVerification @@ -387,8 +391,9 @@ string @@ -400,8 +405,8 @@ bool @@ -413,8 +418,8 @@ string @@ -426,7 +431,8 @@ bool @@ -440,8 +446,8 @@ This option is available only when using the ‘go-git’ GitImplementat @@ -455,7 +461,8 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
-

The repository URL, can be a HTTP/S or SSH address.

+

URL specifies the Git repository URL, can be an HTTP/S or SSH address.
(Optional) -

The secret name containing the Git credentials. -For HTTPS repositories the secret must contain username and password fields. -For SSH repositories the secret must contain ‘identity’, ‘identity.pub’ and ‘known_hosts’ fields.

+

SecretRef specifies the Secret containing authentication credentials for +the GitRepository. +For HTTPS repositories the Secret must contain ‘username’ and ‘password’ +fields. +For SSH repositories the Secret must contain ‘identity’, ‘identity.pub’ +and ‘known_hosts’ fields.
-

The interval at which to check for repository updates.

+

Interval at which to check the GitRepository for updates.
(Optional) -

The timeout for remote Git operations like cloning, defaults to 60s.

+

Timeout for Git operations like cloning, defaults to 60s.
(Optional) -

The Git reference to checkout and monitor for changes, defaults to -master branch.

+

Reference specifies the Git reference to resolve and monitor for +changes, defaults to the ‘master’ branch.
(Optional) -

Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to.

+

Verification specifies the configuration to verify the Git commit +signature(s).
(Optional) -

Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). -If not provided, a default will be used, consult the documentation for your version to find out what those are.

+

Ignore overrides the set of excluded patterns in the .sourceignore format +(which is the same as .gitignore). If not provided, a default will be used, +consult the documentation for your version to find out what those are.
(Optional) -

Suspend tells the controller to suspend the reconciliation of this source. -This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +GitRepository.
(Optional) -

Determines which git client library to use. -Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+

GitImplementation specifies which Git client library implementation to +use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).
(Optional) -

When enabled, after the clone is created, initializes all submodules within, using their default settings. +

RecurseSubmodules enables the initialization of all submodules within +the GitRepository as cloned from the URL, using their default settings. This option is available only when using the ‘go-git’ GitImplementation.
-

Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for -this resource.

+

Include specifies a list of GitRepository resources which Artifacts +should be included in the Artifact produced for this GitRepository.
(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object.
@@ -1206,7 +1213,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus (Appears on: GitRepositorySpec)

-

GitRepositoryInclude defines a source with a from and to path.

+

GitRepositoryInclude specifies a reference to a GitRepository which Artifact +(sub-)contents must be included, and where they should be placed.

@@ -1227,7 +1235,8 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1239,7 +1248,8 @@ string @@ -1251,7 +1261,8 @@ string @@ -1264,7 +1275,7 @@ string (Appears on:GitRepositorySpec)

-

GitRepositoryRef defines the Git ref used for pull and checkout operations.

+

GitRepositoryRef specifies the Git reference to resolve and checkout.

-

Reference to a GitRepository to include.

+

GitRepositoryRef specifies the GitRepository which Artifact contents +must be included.
(Optional) -

The path to copy contents from, defaults to the root directory.

+

FromPath specifies the path to copy contents from, defaults to the root +of the Artifact.
(Optional) -

The path to copy contents to, defaults to the name of the source ref.

+

ToPath specifies the path to copy contents to, defaults to the name of +the GitRepositoryRef.
@@ -1284,7 +1295,9 @@ string @@ -1296,7 +1309,7 @@ string @@ -1308,7 +1321,7 @@ string @@ -1320,7 +1333,10 @@ string @@ -1333,7 +1349,8 @@ string (Appears on:GitRepository)

-

GitRepositorySpec defines the desired state of a Git repository.

+

GitRepositorySpec specifies the required configuration to produce an +Artifact for a Git repository.

(Optional) -

The Git branch to checkout, defaults to master.

+

Branch to checkout, defaults to ‘master’ if no other field is defined.

+

When GitRepositorySpec.GitImplementation is set to ‘go-git’, a shallow +clone of the specified branch is performed.
(Optional) -

The Git tag to checkout, takes precedence over Branch.

+

Tag to checkout, takes precedence over Branch.
(Optional) -

The Git tag semver expression, takes precedence over Tag.

+

SemVer tag expression to checkout, takes precedence over Tag.
(Optional) -

The Git commit SHA to checkout, if specified Tag filters will be ignored.

+

Commit SHA to checkout, takes precedence over all reference fields.

+

When GitRepositorySpec.GitImplementation is set to ‘go-git’, this can be +combined with Branch to shallow clone the branch, in which the commit is +expected to exist.
@@ -1352,7 +1369,7 @@ string @@ -1366,9 +1383,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1381,7 +1401,7 @@ Kubernetes meta/v1.Duration @@ -1395,7 +1415,7 @@ Kubernetes meta/v1.Duration @@ -1409,8 +1429,8 @@ GitRepositoryRef @@ -1424,7 +1444,8 @@ GitRepositoryVerification @@ -1436,8 +1457,9 @@ string @@ -1449,8 +1471,8 @@ bool @@ -1462,8 +1484,8 @@ string @@ -1475,7 +1497,8 @@ bool @@ -1489,8 +1512,8 @@ This option is available only when using the ‘go-git’ GitImplementat @@ -1504,7 +1527,8 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom @@ -1517,7 +1541,7 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom (Appears on:GitRepository)

-

GitRepositoryStatus defines the observed state of a Git repository.

+

GitRepositoryStatus records the observed state of a Git repository.

-

The repository URL, can be a HTTP/S or SSH address.

+

URL specifies the Git repository URL, can be an HTTP/S or SSH address.
(Optional) -

The secret name containing the Git credentials. -For HTTPS repositories the secret must contain username and password fields. -For SSH repositories the secret must contain ‘identity’, ‘identity.pub’ and ‘known_hosts’ fields.

+

SecretRef specifies the Secret containing authentication credentials for +the GitRepository. +For HTTPS repositories the Secret must contain ‘username’ and ‘password’ +fields. +For SSH repositories the Secret must contain ‘identity’, ‘identity.pub’ +and ‘known_hosts’ fields.
-

The interval at which to check for repository updates.

+

Interval at which to check the GitRepository for updates.
(Optional) -

The timeout for remote Git operations like cloning, defaults to 60s.

+

Timeout for Git operations like cloning, defaults to 60s.
(Optional) -

The Git reference to checkout and monitor for changes, defaults to -master branch.

+

Reference specifies the Git reference to resolve and monitor for +changes, defaults to the ‘master’ branch.
(Optional) -

Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to.

+

Verification specifies the configuration to verify the Git commit +signature(s).
(Optional) -

Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). -If not provided, a default will be used, consult the documentation for your version to find out what those are.

+

Ignore overrides the set of excluded patterns in the .sourceignore format +(which is the same as .gitignore). If not provided, a default will be used, +consult the documentation for your version to find out what those are.
(Optional) -

Suspend tells the controller to suspend the reconciliation of this source. -This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +GitRepository.
(Optional) -

Determines which git client library to use. -Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+

GitImplementation specifies which Git client library implementation to +use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).
(Optional) -

When enabled, after the clone is created, initializes all submodules within, using their default settings. +

RecurseSubmodules enables the initialization of all submodules within +the GitRepository as cloned from the URL, using their default settings. This option is available only when using the ‘go-git’ GitImplementation.
-

Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for -this resource.

+

Include specifies a list of GitRepository resources which Artifacts +should be included in the Artifact produced for this GitRepository.
(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object.
@@ -1537,7 +1561,8 @@ int64 @@ -1563,7 +1588,9 @@ string @@ -1577,7 +1604,7 @@ Artifact @@ -1591,7 +1618,8 @@ Artifact @@ -1619,7 +1647,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus (Appears on:GitRepositorySpec)

-

GitRepositoryVerification defines the OpenPGP signature verification process.

+

GitRepositoryVerification specifies the Git commit signature verification +strategy.

(Optional) -

ObservedGeneration is the last observed generation.

+

ObservedGeneration is the last observed generation of the GitRepository +object.
(Optional) -

URL is the fetch link for the artifact output of the last repository sync.

+

URL is the dynamic fetch link for the latest Artifact. +It is provided on a “best effort” basis, and using the precise +GitRepositoryStatus.Artifact data is recommended.
(Optional) -

Artifact represents the output of the last successful repository sync.

+

Artifact represents the last successful GitRepository reconciliation.
(Optional) -

IncludedArtifacts represents the included artifacts from the last successful repository sync.

+

IncludedArtifacts contains a list of the last successfully included +Artifacts as instructed by GitRepositorySpec.Include.
@@ -1638,7 +1667,7 @@ string @@ -1651,7 +1680,8 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference diff --git a/docs/spec/v1beta2/README.md b/docs/spec/v1beta2/README.md index dca99b0e..7e5ab30b 100644 --- a/docs/spec/v1beta2/README.md +++ b/docs/spec/v1beta2/README.md @@ -5,7 +5,7 @@ This is the v1beta2 API specification for defining the desired state sources of ## Specification * Source kinds: - + GitRepository + + [GitRepository](gitrepositories.md) + [HelmRepository](helmrepositories.md) + HelmChart + [Bucket](buckets.md) diff --git a/docs/spec/v1beta2/gitrepositories.md b/docs/spec/v1beta2/gitrepositories.md new file mode 100644 index 00000000..22d54624 --- /dev/null +++ b/docs/spec/v1beta2/gitrepositories.md @@ -0,0 +1,814 @@ +# Git Repositories + +The `GitRepository` API defines a Source to produce an Artifact for a Git +repository revision. + +## Example + +The following is an example of a GitRepository. It creates a tarball +(`.tar.gz`) Artifact with the fetched data from a Git repository for the +resolved reference. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 5m0s + url: https://github.com/stefanprodan/podinfo + ref: + branch: master +``` + +In the above example: + +- A GitRepository named `podinfo` is created, indicated by the + `.metadata.name` field. +- The source-controller checks the Git repository every five minutes, indicated + by the `.spec.interval` field. +- It clones the `master` branch of the `https://github.com/stefanprodan/podinfo` + repository, indicated by the `.spec.ref.branch` and `.spec.url` fields. +- The specified branch and resolved HEAD revision are used as the Artifact + revision, reported in-cluster in the `.status.artifact.revision` field. +- When the current GitRepository revision differs from the latest fetched + revision, a new Artifact is archived. +- The new Artifact is reported in the `.status.artifact` field. + +You can run this example by saving the manifest into `gitrepository.yaml`. + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f gitrepository.yaml + ``` + +2. Run `kubectl get gitrepository` to see the GitRepository: + + ```console + NAME URL READY STATUS AGE + podinfo https://github.com/stefanprodan/podinfo True stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' 5s + ``` + +3. Run `kubectl describe gitrepository podinfo` to see the [Artifact](#artifact) + and [Conditions](#conditions) in the GitRepository's Status: + + ```console + ... + Status: + Artifact: + Checksum: 95e386f421272710c4cedbbd8607dbbaa019d500e7a5a0b6720bc7bebefc7bf2 + Last Update Time: 2022-02-14T11:23:36Z + Path: gitrepository/default/podinfo/132f4e719209eb10b9485302f8593fc0e680f4fc.tar.gz + Revision: master/132f4e719209eb10b9485302f8593fc0e680f4fc + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/podinfo/132f4e719209eb10b9485302f8593fc0e680f4fc.tar.gz + Conditions: + Last Transition Time: 2022-02-14T11:23:36Z + Message: stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + Observed Generation: 1 + Reason: Succeeded + Status: True + Type: Ready + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/podinfo/latest.tar.gz + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal GitOperationSucceed 62s source-controller cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + Normal NewArtifact 62s source-controller stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + ``` + +## Writing a GitRepository spec + +As with all other Kubernetes config, a GitRepository needs `apiVersion`, +`kind`, and `metadata` fields. The name of a GitRepository object must be a +valid [DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names#dns-subdomain-names). + +A GitRepository also needs a +[`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). + +### URL + +`.spec.url` is a required field that specifies the HTTP/S or SSH address of the +Git repository. + +**Note:** Unlike using `git`, the +[shorter scp-like syntax](https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols#_the_ssh_protocol) +is not supported for SSH addresses (e.g. `user@example.com:repository.git`). +Instead, the valid URL format is `ssh://user@example.com:22/repository.git`. + +### Secret reference + +`.spec.secretRef.name` is an optional field to specify a name reference to a +Secret in the same namespace as the GitRepository, containing authentication +credentials for the Git repository. + +The required fields in the Secret depend on the specified protocol in the +[URL](#url). + +#### Basic access authentication + +To authenticate towards a Git repository over HTTPS using basic access +authentication (in other words: using a username and password), the referenced +Secret is expected to contain `.data.username` and `.data.password` values. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: basic-access-auth +type: Opaque +data: + username: + password: +``` + +#### HTTPS Certificate Authority + +To provide a Certificate Authority to trust while connecting with a Git +repository over HTTPS, the referenced Secret can contain a `.data.caFile` +value. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-ca-credentials + namespace: default +type: Opaque +data: + caFile: +``` + +#### SSH authentication + +To authenticate towards a Git repository over SSH, the referenced Secret is +expected to contain `.data.identity`, `.data.identity.pub` and `known_hosts` +fields. With the respective private and public key of the SSH key pair, and the +host keys of the Git repository. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssh-credentials +type: Opaque +data: + identity: + identity.pub: + known_hosts: +``` + +### Interval + +`.spec.interval` is a required field that specifies the interval at which the +Git repository must be fetched. + +After successfully reconciling the object, the source-controller requeues it +for inspection after the specified interval. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to reconcile the object every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. a change to +the spec), this is handled instantly outside the interval window. + +### Timeout + +`.spec.timeout` is an optional field to specify a timeout for Git operations +like cloning. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `1m30s` for a timeout of one minute and thirty seconds. The default value +is `60s`. + +### Reference + +`.spec.ref` is an optional field to specify the Git reference to resolve and +watch for changes. References are specified in one or more subfields +(`.branch`, `.tag`, `.semver`, `.commit`), with latter listed fields taking +precedence over earlier ones. If not specified, it defaults to a `master` +branch reference. + +#### Branch example + +To Git checkout a specified branch, use `.spec.ref.branch`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + branch: +``` + +Using the [`go-git` Git implementation](#git-implementation), this will perform +a shallow clone to only fetch the specified branch. + +#### Tag example + +To Git checkout a specified tag, use `.spec.ref.tag`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + tag: +``` + +This field takes precedence over [`.branch`](#branch-example). + +#### SemVer example + +To Git checkout a tag based on a +[SemVer range](https://github.com/Masterminds/semver#checking-version-constraints), +use `.spec.ref.semver`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + # SemVer range reference: https://github.com/Masterminds/semver#checking-version-constraints + semver: "" +``` + +This field takes precedence over [`.branch`](#branch-example) and +[`.tag`](#tag-example). + +#### Commit example + +To Git checkout a specified commit, use `.spec.ref.commit`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + commit: "" +``` + +This field takes precedence over all other fields. Using the [`go-git` Git +implementation](#git-implementation), it can be combined with `.spec.ref.branch` +to perform a shallow clone of the branch, in which the commit must exist: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + branch: + commit: "" +``` + +### Verification + +`.spec.verify` is an optional field to enable the verification of Git commit +signatures. The field offers two subfields: + +- `.mode`, to specify what Git commit object should be verified. Only supports + `head` at present. +- `.secretRef.name`, to specify a reference to a Secret in the same namespace as + the GitRepository. Containing the (PGP) public keys of trusted Git authors. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master + verify: + mode: head + secretRef: + name: pgp-public-keys +``` + +When the verification succeeds, the controller adds a Condition with the +following attributes to the GitRepository's `.status.conditions`: + +- `type: SourceVerifiedCondition` +- `status: "True"` +- `reason: Succeeded` + +#### Verification Secret example + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: pgp-public-keys + namespace: default +type: Opaque +data: + author1.asc: + author2.asc: +``` + +Exporting armored public keys (`.asc` files) using `gpg`, and generating a +Secret: + +```sh +# Export armored public keys +gpg --export --armor 3CB12BA185C47B67 > author1.asc +gpg --export --armor 6A7436E8790F8689 > author2.asc +# Generate secret +kubectl create secret generic pgp-public-keys \ + --from-file=author1.asc \ + --from-file=author2.asc \ + -o yaml +``` + +### Ignore + +`.spec.ignore` is an optional field to specify rules in [the `.gitignore` +pattern format](https://git-scm.com/docs/gitignore#_pattern_format). Paths +matching the defined rules are excluded while archiving. + +When specified, `.spec.ignore` overrides the [default exclusion +list](#default-exclusions), and may overrule the [`.sourceignore` file +exclusions](#sourceignore-file). See [excluding files](#excluding-files) +for more information. + +### Suspend + +`.spec.suspend` is an optional field to suspend the reconciliation of a +GitRepository. When set to `true`, the controller will stop reconciling the +GitRepository, and changes to the resource or in the Git repository will not +result in a new Artifact. When the field is set to `false` or removed, it will +resume. + +### Git implementation + +`.spec.gitImplementation` is an optional field to change the client library +implementation used for Git operations (e.g. clone, checkout). The default +value is `go-git`. + +Unless you need support for a specific Git wire protocol functionality not +supported by the default implementation (as documented below), changing the +implementation is generally not recommended as it can come with its own set of +drawbacks. For example, not being able to make use of shallow clones forces the +controller to fetch the whole Git history tree instead of a specific one, +resulting in an increase of disk space and traffic usage. + +| Git Implementation | Shallow Clones | Git Submodules | V2 Protocol Support | +|--------------------|----------------|----------------|---------------------| +| `go-git` | true | true | false | +| `libgit2` | false | false | true | + +Some Git providers like Azure DevOps _require_ the `libgit2` implementation, as +their Git servers provide only support for the +[v2 protocol](https://git-scm.com/docs/protocol-v2). + +#### Proxy support + +When a proxy is configured in the source-controller Pod through the appropriate +environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. There may be +some limitations in the proxy support based on the Git implementation. + +| Git Implementation | HTTP_PROXY | HTTPS_PROXY | NO_PROXY | Self-signed Certs | +|--------------------|------------|-------------|----------|-------------------| +| `go-git` | true | true | true | false |n +| `libgit2` | false | true | true | true | + +### Recurse submodules + +`.spec.recurseSubmodules` is an optional field to enable the initialization of +all submodules within the cloned Git repository, using their default settings. +This option is only available when using the (default) `go-git` [Git +implementation](#git-implementation), and defaults to `false`. + +Note that for most Git providers (e.g. GitHub and GitLab), deploy keys can not +be used as reusing a key across multiple repositories is not allowed. You have +to use either [HTTPS token-based authentication](#basic-access-authentication), +or an SSH key belonging to a (bot) user who has access to the main repository +and all submodules. + +### Include + +`.spec.include` is an optional field to map the contents of GitRepository +Artifacts into another. This may look identical to Git submodules but has +multiple benefits over regular submodules: + +- Including a `GitRepository` allows you to use different authentication + methods for different repositories. +- A change in the included repository will trigger an update of the including + repository. +- Multiple `GitRepository` objects could include the same repository, which + decreases the amount of cloning done compared to using submodules. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: include-example +spec: + include: + - repository: + name: other-repository + fromPath: deploy/kubernetes + toPath: base/app +``` + +The `.fromPath` and `.toPath` fields allow you to limit the files included, and +where they will be copied to. If you do not specify a value for `.fromPath`, +all files from the referenced GitRepository Artifact will be included. The +`.toPath` defaults to the `.repository.name` (e.g. `./other-repository/*`). + +## Working with GitRepositories + +### Excluding files + +By default, files which match the [default exclusion rules](#default-exclusions) +are excluded while archiving the Git repository contents as an Artifact. It is +possible to overwrite and/or overrule the default exclusions using a file in +the Git repository and/or an in-spec set of rules. + +#### `.sourceignore` file + +Excluding files is possible by adding a `.sourceignore` file in the Git +repository. The `.sourceignore` file follows [the `.gitignore` pattern +format](https://git-scm.com/docs/gitignore#_pattern_format), and +pattern entries may overrule [default exclusions](#default-exclusions). + +#### Ignore spec + +Another option is to define the exclusions within the GitRepository spec, using +the [`.spec.ignore` field](#ignore). Specified rules override the [default +exclusion list](#default-exclusions), and may overrule `.sourceignore` file +exclusions. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ignore: | + # exclude all + /* + # include deploy dir + !/deploy + # exclude file extensions from deploy dir + /deploy/**/*.md + /deploy/**/*.txt +``` + +### Triggering a reconcile + +To manually tell the source-controller to reconcile a GitRepository outside the +[specified interval window](#interval), a GitRepository can be annotated with +`reconcile.fluxcd.io/requestedAt: `. Annotating the resource +queues the GitRepository for reconciliation if the `` differs +from the last value the controller acted on, as reported in +[`.status.lastHandledReconcileAt`](#last-handled-reconcile-at). + +Using `kubectl`: + +```sh +kubectl annotate --overwrite gitrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +``` + +Using `flux`: + +```sh +flux reconcile source git +``` + +### Waiting for `Ready` + +When a change is applied, it is possible to wait for the GitRepository to reach +a [ready state](#ready-gitrepository) using `kubectl`: + +```sh +kubectl wait gitrepository/ --for=condition=ready --timeout=1m +``` + +### Suspending and resuming + +When you find yourself in a situation where you temporarily want to pause the +reconciliation of a GitRepository, you can suspend it using the +[`.spec.suspend` field](#suspend). + +#### Suspend a GitRepository + +In your YAML declaration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + suspend: true +``` + +Using `kubectl`: + +```sh +kubectl patch gitrepository -p '{\"spec\": {\"suspend\" : true }}' +``` + +Using `flux`: + +```sh +flux suspend source git +``` + +**Note:** When a GitRepository has an Artifact and is suspended, and this +Artifact later disappears from the storage due to e.g. the source-controller +Pod being evicted from a Node, this will not be reflected in the +GitRepository's Status until it is resumed. + +#### Resume a GitRepository + +In your YAML declaration, comment out (or remove) the field: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + # suspend: true +``` + +**Note:** Setting the field value to `false` has the same effect as removing +it, but does not allow for "hot patching" using e.g. `kubectl` while practicing +GitOps; as the manually applied patch would be overwritten by the declared +state in Git. + +Using `kubectl`: + +```sh +kubectl patch gitrepository -p '{\"spec\" : {\"suspend\" : false }}' +``` + +Using `flux`: + +```sh +flux resume source git +``` + +### Debugging a GitRepository + +There are several ways to gather information about a GitRepository for +debugging purposes. + +#### Describe the GitRepository + +Describing a GitRepository using +`kubectl describe gitrepository ` +displays the latest recorded information for the resource in the `Status` and +`Events` sections: + +```console +... +Status: +... + Conditions: + Last Transition Time: 2022-02-14T09:40:27Z + Message: reconciling new generation 2 + Observed Generation: 2 + Reason: NewGeneration + Status: True + Type: Reconciling + Last Transition Time: 2022-02-14T09:40:27Z + Message: failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" + Observed Generation: 2 + Reason: GitOperationFailed + Status: False + Type: Ready + Last Transition Time: 2022-02-14T09:40:27Z + Message: failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" + Observed Generation: 2 + Reason: GitOperationFailed + Status: True + Type: FetchFailed + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/gitrepository-sample/latest.tar.gz +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning GitOperationFailed 2s (x9 over 4s) source-controller failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" +``` + +#### Trace emitted Events + +To view events for specific GitRepository(s), `kubectl get events` can be used +in combination with `--field-sector` to list the Events for specific objects. +For example, running + +```sh +kubectl get events --field-selector involvedObject.kind=GitRepository,involvedObject.name= +``` + +lists + +```console +LAST SEEN TYPE REASON OBJECT MESSAGE +2m14s Normal GitOperationSucceed gitrepository/ cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' +2m14s Normal NewArtifact gitrepository/ stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' +94s Warning GitOperationFailed gitrepository/gitrepository-sample failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" +``` + +Besides being reported in Events, the reconciliation errors are also logged by +the controller. The Flux CLI offer commands for filtering the logs for a +specific GitRepository, e.g. +`flux logs --level=error --kind=GitRepository --name=`. + +## GitRepository Status + +### Artifact + +The GitRepository reports the latest synchronized state from the Git repository +as an Artifact object in the `.status.artifact` of the resource. + +The Artifact file is a gzip compressed TAR archive (`.tar.gz`), and +can be retrieved in-cluster from the `.status.artifact.url` HTTP address. + +#### Artifact example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +status: + artifact: + checksum: e750c7a46724acaef8f8aa926259af30bbd9face2ae065ae8896ba5ee5ab832b + lastUpdateTime: "2022-01-29T06:59:23Z" + path: gitrepository///c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2.tar.gz + revision: master/363a6a8fe6a7f13e05d34c163b0ef02a777da20a + url: http://source-controller..svc.cluster.local./gitrepository///363a6a8fe6a7f13e05d34c163b0ef02a777da20a.tar.gz +``` + +#### Default exclusions + +The following files and extensions are excluded from the Artifact by +default: + +- Git files (`.git/, .gitignore, .gitmodules, .gitattributes`) +- File extensions (`.jpg, .jpeg, .gif, .png, .wmv, .flv, .tar.gz, .zip`) +- CI configs (`.github/, .circleci/, .travis.yml, .gitlab-ci.yml, appveyor.yml, .drone.yml, cloudbuild.yaml, codeship-services.yml, codeship-steps.yml`) +- CLI configs (`.goreleaser.yml, .sops.yaml`) +- Flux v1 config (`.flux.yaml`) + +To define your own exclusion rules, see [excluding files](#excluding-files). + +### Conditions + +A GitRepository enters various states during its lifecycle, reflected as +[Kubernetes Conditions][typical-status-properties]. +It can be [reconciling](#reconciling-gitrepository) while fetching the Git +state, it can be [ready](#ready-gitrepository), or it can [fail during +reconciliation](#failed-gitrepository). + +The GitRepository API is compatible with the [kstatus specification][kstatus-spec], +and reports `Reconciling` and `Stalled` conditions where applicable to +provide better (timeout) support to solutions polling the GitRepository to +become `Ready`. + +#### Reconciling GitRepository + +The source-controller marks a GitRepository as _reconciling_ when one of the +following is true: + +- There is no current Artifact for the GitRepository, or the reported Artifact + is determined to have disappeared from the storage. +- The generation of the GitRepository is newer than the [Observed + Generation](#observed-generation). +- The newly resolved Artifact revision differs from the current Artifact. + +When the GitRepository is "reconciling", the `Ready` Condition status becomes +`False`, and the controller adds a Condition with the following attributes to +the GitRepository's `.status.conditions`: + +- `type: Reconciling` +- `status: "True"` +- `reason: NewGeneration` | `reason: NoArtifact` | `reason: NewRevision` + +If the reconciling state is due to a new revision, an additional Condition is +added with the following attributes: + +- `type: ArtifactOutdated` +- `status: "True"` +- `reason: NewRevision` + +Both Conditions have a ["negative polarity"][typical-status-properties], +and are only present on the GitRepository while their status value is `"True"`. + +#### Ready GitRepository + +The source-controller marks a GitRepository as _ready_ when it has the +following characteristics: + +- The GitRepository reports an [Artifact](#artifact). +- The reported Artifact exists in the controller's Artifact storage. +- The controller was able to communicate with the remote Git repository using + the current spec. +- The revision of the reported Artifact is up-to-date with the latest + resolved revision of the remote Git repository. + +When the GitRepository is "ready", the controller sets a Condition with the +following attributes in the GitRepository's `.status.conditions`: + +- `type: Ready` +- `status: "True"` +- `reason: Succeeded` + +This `Ready` Condition will retain a status value of `"True"` until the +GitRepository is marked as [reconciling](#reconciling-gitrepository), or e.g. a +[transient error](#failed-gitrepository) occurs due to a temporary network issue. + +#### Failed GitRepository + +The source-controller may get stuck trying to produce an Artifact for a +GitRepository without completing. This can occur due to some of the following +factors: + +- The remote Git repository [URL](#url) is temporarily unavailable. +- The Git repository does not exist. +- The [Secret reference](#secret-reference) contains a reference to a + non-existing Secret. +- A specified Include is unavailable. +- The verification of the Git commit signature failed. +- The credentials in the referenced Secret are invalid. +- The GitRepository spec contains a generic misconfiguration. + +When this happens, the controller sets the `Ready` Condition status to `False`, +and adds a Condition with the following attributes to the GitRepository's +`.status.conditions`: + +- `type: FetchFailed` | `type: IncludeUnavailableCondition` +- `status: "True"` +- `reason: AuthenticationFailed` | `reason: GitOperationFailed` | `reason: StorageOperationFailed` + +This condition has a ["negative polarity"][typical-status-properties], +and is only present on the GitRepository while the status value is `"True"`. + +In addition to the above Condition types, when the +[verification of a Git commit signature](#verification) fails. A condition with +the following attributes is added to the GitRepository's `.status.conditions`: + +- `type: SourceVerifiedCondition` +- `status: "False"` +- `reason: Failed` + +While the GitRepository has one or more of these Conditions, the controller +will continue to attempt to produce an Artifact for the resource with an +exponential backoff, until it succeeds and the GitRepository is marked as +[ready](#ready-gitrepository). + +Note that a GitRepository can be [reconciling](#reconciling-gitrepository) +while failing at the same time, for example due to a newly introduced +configuration issue in the GitRepository spec. + +### Observed Generation + +The source-controller reports an [observed generation][typical-status-properties] +in the GitRepository's `.status.observedGeneration`. The observed generation is +the latest `.metadata.generation` which resulted in either a [ready state](#ready-gitrepository), +or stalled due to error it can not recover from without human +intervention. + +### Last Handled Reconcile At + +The source-controller reports the last `reconcile.fluxcd.io/requestedAt` +annotation value it acted on in the `.status.lastHandledReconcileAt` field. + +For practical information about this field, see [triggering a +reconcile](#triggering-a-reconcile). + +[typical-status-properties]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[kstatus-spec]: https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus
-

Mode describes what Git object should be verified, currently (‘head’).

+

Mode specifies what Git object should be verified, currently (‘head’).
-

SecretRef containing the public keys of all trusted Git authors.

+

SecretRef specifies the Secret containing the public keys of trusted Git +authors.