diff --git a/api/v1alpha1/condition_types.go b/api/v1alpha1/condition_types.go index 7c5136dc..cbfecb1f 100644 --- a/api/v1alpha1/condition_types.go +++ b/api/v1alpha1/condition_types.go @@ -43,4 +43,8 @@ const ( // URLInvalidReason represents the fact that a given source has an invalid URL. URLInvalidReason string = "URLInvalid" + + // AuthenticationFailedReason represents the fact that a given secret doesn't + // have the required fields or the provided credentials don't match. + AuthenticationFailedReason string = "AuthenticationFailed" ) diff --git a/api/v1alpha1/gitrepository_types.go b/api/v1alpha1/gitrepository_types.go index 8d19af63..5ae44276 100644 --- a/api/v1alpha1/gitrepository_types.go +++ b/api/v1alpha1/gitrepository_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,11 +28,24 @@ type GitRepositorySpec struct { // +required URL string `json:"url"` + // The secret name containing the Git credentials. + // For HTTPS repositories the secret must contain username and password fields. + // For SSH repositories the secret must contain identity, identity.pub and known_hosts fields. + // +optional + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + // The interval at which to check for repository updates. // +required Interval metav1.Duration `json:"interval"` - // The git branch to checkout, defaults to ('master'). + // The git reference to checkout and monitor for changes, defaults to master branch. + // +optional + Reference *GitRepositoryRef `json:"ref,omitempty"` +} + +// GitRepositoryRef defines the git ref used for pull and checkout operations +type GitRepositoryRef struct { + // The git branch to checkout, defaults to master. // +optional Branch string `json:"branch"` @@ -42,6 +56,10 @@ type GitRepositorySpec struct { // The git tag semver expression, takes precedence over tag. // +optional SemVer string `json:"semver"` + + // The git commit sha to checkout, if specified tag filters will be ignored. + // +optional + Commit string `json:"commit"` } // GitRepositoryStatus defines the observed state of GitRepository diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 900570a4..ded75ecb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -29,7 +30,7 @@ func (in *GitRepository) DeepCopyInto(out *GitRepository) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -83,10 +84,35 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitRepositoryRef) DeepCopyInto(out *GitRepositoryRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositoryRef. +func (in *GitRepositoryRef) DeepCopy() *GitRepositoryRef { + if in == nil { + return nil + } + out := new(GitRepositoryRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitRepositorySpec) DeepCopyInto(out *GitRepositorySpec) { *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } out.Interval = in.Interval + if in.Reference != nil { + in, out := &in.Reference, &out.Reference + *out = new(GitRepositoryRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositorySpec. diff --git a/config/crd/bases/source.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.fluxcd.io_gitrepositories.yaml index 357d70ab..92c7cf37 100644 --- a/config/crd/bases/source.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.fluxcd.io_gitrepositories.yaml @@ -49,18 +49,39 @@ spec: spec: description: GitRepositorySpec defines the desired state of GitRepository properties: - branch: - description: The git branch to checkout, defaults to ('master'). - type: string interval: description: The interval at which to check for repository updates. type: string - semver: - description: The git tag semver expression, takes precedence over tag. - type: string - tag: - description: The git tag to checkout, takes precedence over branch. - type: string + ref: + description: The git reference to checkout and monitor for changes, + defaults to master branch. + properties: + branch: + description: The git branch to checkout, defaults to master. + type: string + commit: + description: The git commit sha to checkout, if specified tag filters + will be ignored. + type: string + semver: + description: The git tag semver expression, takes precedence over + tag. + type: string + tag: + description: The git tag to checkout, takes precedence over branch. + type: string + type: object + secretRef: + description: The secret name containing the Git credentials. For HTTPS + repositories the secret must contain username and password fields. + For SSH repositories the secret must contain identity, identity.pub + and known_hosts fields. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object url: description: The repository URL, can be a HTTP or SSH address. pattern: ^(http|https|ssh):// diff --git a/config/samples/source_v1alpha1_gitrepository.yaml b/config/samples/source_v1alpha1_gitrepository.yaml index df84d650..673748f9 100644 --- a/config/samples/source_v1alpha1_gitrepository.yaml +++ b/config/samples/source_v1alpha1_gitrepository.yaml @@ -7,5 +7,5 @@ metadata: spec: interval: 1m url: https://github.com/stefanprodan/podinfo - branch: master - tag: "3.2.2" + ref: + branch: master diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 2d82c1a1..5b38bfd2 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -21,16 +21,21 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "time" "github.com/blang/semver" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -126,95 +131,146 @@ func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourcev1.SourceCondition, string, error) { + // set defaults: master branch, no tags fetching, max two commits + branch := "master" + tagMode := git.NoTags + depth := 2 + // determine ref - refName := plumbing.NewBranchReferenceName("master") - if repository.Spec.Branch != "" { - refName = plumbing.NewBranchReferenceName(repository.Spec.Branch) - } - if repository.Spec.Tag != "" { - refName = plumbing.NewTagReferenceName(repository.Spec.Tag) + refName := plumbing.NewBranchReferenceName(branch) + if repository.Spec.Reference != nil { + if repository.Spec.Reference.Branch != "" { + branch = repository.Spec.Reference.Branch + refName = plumbing.NewBranchReferenceName(branch) + } + if repository.Spec.Reference.Commit != "" { + depth = 0 + } else { + if repository.Spec.Reference.Tag != "" { + refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag) + } + if repository.Spec.Reference.SemVer != "" { + tagMode = git.AllTags + } + } } - // create tmp dir - dir, err := ioutil.TempDir("", repository.Name) + // create tmp dir for SSH known_hosts + tmpSSH, err := ioutil.TempDir("", repository.Name) if err != nil { - err = fmt.Errorf("tmp dir error %w", err) + err = fmt.Errorf("tmp dir error: %w", err) return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err } - defer os.RemoveAll(dir) + defer os.RemoveAll(tmpSSH) + + auth, err := r.auth(repository, tmpSSH) + if err != nil { + err = fmt.Errorf("auth error: %w", err) + return NotReadyCondition(sourcev1.AuthenticationFailedReason, err.Error()), "", err + } + + // create tmp dir for the Git clone + tmpGit, err := ioutil.TempDir("", repository.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + defer os.RemoveAll(tmpGit) // clone to tmp - repo, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: repository.Spec.URL, - Depth: 2, - ReferenceName: refName, - SingleBranch: true, - Tags: git.AllTags, + repo, err := git.PlainClone(tmpGit, false, &git.CloneOptions{ + URL: repository.Spec.URL, + Auth: auth, + RemoteName: "origin", + ReferenceName: refName, + SingleBranch: true, + NoCheckout: false, + Depth: depth, + RecurseSubmodules: 0, + Progress: nil, + Tags: tagMode, }) if err != nil { - err = fmt.Errorf("git clone error %w", err) + err = fmt.Errorf("git clone error: %w", err) return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err } - // checkout tag based on semver expression - if repository.Spec.SemVer != "" { - rng, err := semver.ParseRange(repository.Spec.SemVer) - if err != nil { - err = fmt.Errorf("semver parse range error %w", err) - return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err - } - - repoTags, err := repo.Tags() - if err != nil { - err = fmt.Errorf("git list tags error %w", err) - return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err - } - - tags := make(map[string]string) - _ = repoTags.ForEach(func(t *plumbing.Reference) error { - tags[t.Name().Short()] = t.Strings()[1] - return nil - }) - - svTags := make(map[string]string) - svers := []semver.Version{} - for tag, _ := range tags { - v, _ := semver.ParseTolerant(tag) - if rng(v) { - svers = append(svers, v) - svTags[v.String()] = tag - } - } - - if len(svers) > 0 { - semver.Sort(svers) - v := svers[len(svers)-1] - t := svTags[v.String()] - commit := tags[t] - + // checkout commit or tag + if repository.Spec.Reference != nil { + if commit := repository.Spec.Reference.Commit; commit != "" { w, err := repo.Worktree() if err != nil { - err = fmt.Errorf("git worktree error %w", err) + err = fmt.Errorf("git worktree error: %w", err) return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err } err = w.Checkout(&git.CheckoutOptions{ - Hash: plumbing.NewHash(commit), + Hash: plumbing.NewHash(commit), + Force: true, }) if err != nil { - err = fmt.Errorf("git checkout error %w", err) + err = fmt.Errorf("git checkout %s for %s error: %w", commit, branch, err) + return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err + } + } else if exp := repository.Spec.Reference.SemVer; exp != "" { + rng, err := semver.ParseRange(exp) + if err != nil { + err = fmt.Errorf("semver parse range error: %w", err) + return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err + } + + repoTags, err := repo.Tags() + if err != nil { + err = fmt.Errorf("git list tags error: %w", err) + return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err + } + + tags := make(map[string]string) + _ = repoTags.ForEach(func(t *plumbing.Reference) error { + tags[t.Name().Short()] = t.Strings()[1] + return nil + }) + + svTags := make(map[string]string) + svers := []semver.Version{} + for tag, _ := range tags { + v, _ := semver.ParseTolerant(tag) + if rng(v) { + svers = append(svers, v) + svTags[v.String()] = tag + } + } + + if len(svers) > 0 { + semver.Sort(svers) + v := svers[len(svers)-1] + t := svTags[v.String()] + commit := tags[t] + + w, err := repo.Worktree() + if err != nil { + err = fmt.Errorf("git worktree error: %w", err) + return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err + } + + err = w.Checkout(&git.CheckoutOptions{ + Hash: plumbing.NewHash(commit), + }) + if err != nil { + err = fmt.Errorf("git checkout error: %w", err) + return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err + } + } else { + err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer) return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err } - } else { - err = fmt.Errorf("no match found for semver %s", repository.Spec.SemVer) - return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err } } // read commit hash ref, err := repo.Head() if err != nil { - err = fmt.Errorf("git resolve HEAD error %w", err) + err = fmt.Errorf("git resolve HEAD error: %w", err) return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err } @@ -224,7 +280,7 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc // create artifact dir err = r.Storage.MkdirAll(artifact) if err != nil { - err = fmt.Errorf("mkdir dir error %w", err) + err = fmt.Errorf("mkdir dir error: %w", err) return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err } @@ -237,13 +293,20 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc defer unlock() // archive artifact - err = r.Storage.Archive(artifact, dir, "") + err = r.Storage.Archive(artifact, tmpGit, "") if err != nil { - err = fmt.Errorf("storage error %w", err) + err = fmt.Errorf("storage archive error: %w", err) return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err } - message := fmt.Sprintf("Artifact is available at %s", artifact.Path) + // update latest symlink + err = r.Storage.Symlink(artifact, "latest.tar.gz") + if err != nil { + err = fmt.Errorf("storage lock error: %w", err) + return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + + message := fmt.Sprintf("Artifact is available at: %s", artifact.Path) return ReadyCondition(sourcev1.GitOperationSucceedReason, message), artifact.URL, nil } @@ -283,3 +346,74 @@ func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) { } } } + +func (r *GitRepositoryReconciler) auth(repository sourcev1.GitRepository, tmp string) (transport.AuthMethod, error) { + if repository.Spec.SecretRef == nil { + return nil, nil + } + + name := types.NamespacedName{ + Namespace: repository.GetNamespace(), + Name: repository.Spec.SecretRef.Name, + } + + var secret corev1.Secret + err := r.Client.Get(context.TODO(), name, &secret) + if err != nil { + return nil, err + } + + credentials := secret.Data + + // HTTP auth + if strings.HasPrefix(repository.Spec.URL, "http") { + auth := &http.BasicAuth{} + if username, ok := credentials["username"]; ok { + auth.Username = string(username) + } + if password, ok := credentials["password"]; ok { + auth.Password = string(password) + } + + if auth.Username == "" || auth.Password == "" { + return nil, fmt.Errorf("invalid '%s' secret data: required fields username and password", + repository.Spec.SecretRef.Name) + } + + return auth, nil + } + + // SSH auth + if strings.HasPrefix(repository.Spec.URL, "ssh") { + var privateKey []byte + if identity, ok := credentials["identity"]; ok { + privateKey = identity + } else { + return nil, fmt.Errorf("invalid '%s' secret data: required field identity", repository.Spec.SecretRef.Name) + } + + pk, err := ssh.NewPublicKeys("git", privateKey, "") + if err != nil { + return nil, err + } + + known_hosts := filepath.Join(tmp, "known_hosts") + if kh, ok := credentials["known_hosts"]; ok { + if err := ioutil.WriteFile(filepath.Join(tmp, "known_hosts"), kh, 0644); err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("invalid '%s' secret data: required field known_hosts", repository.Spec.SecretRef.Name) + } + + callback, err := ssh.NewKnownHostsCallback(known_hosts) + if err != nil { + return nil, err + } + pk.HostKeyCallback = callback + + return pk, nil + } + + return nil, nil +} diff --git a/controllers/storage.go b/controllers/storage.go index f52ea249..b42c5d49 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -95,7 +95,7 @@ func (s *Storage) RemoveAllButCurrent(artifact Artifact) error { dir := filepath.Dir(artifact.Path) errors := []string{} _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if path != artifact.Path && !info.IsDir() { + if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink { if err := os.Remove(path); err != nil { errors = append(errors, info.Name()) } @@ -149,11 +149,33 @@ func (s *Storage) WriteFile(artifact Artifact, data []byte) error { return ioutil.WriteFile(artifact.Path, data, 0644) } +// Symlink creates or updates a symbolic link for the given artifact +func (s *Storage) Symlink(artifact Artifact, linkName string) error { + dir := filepath.Dir(artifact.Path) + link := filepath.Join(dir, linkName) + tmpLink := link + ".tmp" + + if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { + return err + } + + if err := os.Symlink(artifact.Path, tmpLink); err != nil { + return err + } + + if err := os.Rename(tmpLink, link); err != nil { + return err + } + + return nil +} + // Checksum returns the SHA1 checksum for the given bytes as a string func (s *Storage) Checksum(b []byte) string { return fmt.Sprintf("%x", sha1.Sum(b)) } +// Lock creates a file lock for the given artifact func (s *Storage) Lock(artifact Artifact) (unlock func(), err error) { lockFile := artifact.Path + ".lock" mutex := lockedfile.MutexAt(lockFile)