Merge pull request #6 from fluxcd/git-auth

Implement Git authentication

api/v1alpha1/condition_types.go

@@ -43,4 +43,8 @@ const (
 
 	// URLInvalidReason represents the fact that a given source has an invalid URL.
 	URLInvalidReason string = "URLInvalid"
+
+	// AuthenticationFailedReason represents the fact that a given secret doesn't
+	// have the required fields or the provided credentials don't match.
+	AuthenticationFailedReason string = "AuthenticationFailed"
 )

api/v1alpha1/gitrepository_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -27,11 +28,24 @@ type GitRepositorySpec struct {
 	// +required
 	URL string `json:"url"`
 
+	// The secret name containing the Git credentials.
+	// For HTTPS repositories the secret must contain username and password fields.
+	// For SSH repositories the secret must contain identity, identity.pub and known_hosts fields.
+	// +optional
+	SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+
 	// The interval at which to check for repository updates.
 	// +required
 	Interval metav1.Duration `json:"interval"`
 
-	// The git branch to checkout, defaults to ('master').
+	// The git reference to checkout and monitor for changes, defaults to master branch.
+	// +optional
+	Reference *GitRepositoryRef `json:"ref,omitempty"`
+}
+
+// GitRepositoryRef defines the git ref used for pull and checkout operations
+type GitRepositoryRef struct {
+	// The git branch to checkout, defaults to master.
 	// +optional
 	Branch string `json:"branch"`
 
@@ -42,6 +56,10 @@ type GitRepositorySpec struct {
 	// The git tag semver expression, takes precedence over tag.
 	// +optional
 	SemVer string `json:"semver"`
+
+	// The git commit sha to checkout, if specified tag filters will be ignored.
+	// +optional
+	Commit string `json:"commit"`
 }
 
 // GitRepositoryStatus defines the observed state of GitRepository

api/v1alpha1/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -29,7 +30,7 @@ func (in *GitRepository) DeepCopyInto(out *GitRepository) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
 }
 
@@ -83,10 +84,35 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitRepositoryRef) DeepCopyInto(out *GitRepositoryRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositoryRef.
+func (in *GitRepositoryRef) DeepCopy() *GitRepositoryRef {
+	if in == nil {
+		return nil
+	}
+	out := new(GitRepositoryRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitRepositorySpec) DeepCopyInto(out *GitRepositorySpec) {
 	*out = *in
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(v1.LocalObjectReference)
+		**out = **in
+	}
 	out.Interval = in.Interval
+	if in.Reference != nil {
+		in, out := &in.Reference, &out.Reference
+		*out = new(GitRepositoryRef)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositorySpec.

config/crd/bases/source.fluxcd.io_gitrepositories.yaml

@@ -49,18 +49,39 @@ spec:
         spec:
           description: GitRepositorySpec defines the desired state of GitRepository
           properties:
-            branch:
-              description: The git branch to checkout, defaults to ('master').
-              type: string
             interval:
               description: The interval at which to check for repository updates.
               type: string
-            semver:
+            ref:
-              description: The git tag semver expression, takes precedence over tag.
+              description: The git reference to checkout and monitor for changes,
-              type: string
+                defaults to master branch.
-            tag:
+              properties:
-              description: The git tag to checkout, takes precedence over branch.
+                branch:
-              type: string
+                  description: The git branch to checkout, defaults to master.
+                  type: string
+                commit:
+                  description: The git commit sha to checkout, if specified tag filters
+                    will be ignored.
+                  type: string
+                semver:
+                  description: The git tag semver expression, takes precedence over
+                    tag.
+                  type: string
+                tag:
+                  description: The git tag to checkout, takes precedence over branch.
+                  type: string
+              type: object
+            secretRef:
+              description: The secret name containing the Git credentials. For HTTPS
+                repositories the secret must contain username and password fields.
+                For SSH repositories the secret must contain identity, identity.pub
+                and known_hosts fields.
+              properties:
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                    TODO: Add other useful fields. apiVersion, kind, uid?'
+                  type: string
+              type: object
             url:
               description: The repository URL, can be a HTTP or SSH address.
               pattern: ^(http|https|ssh)://

config/samples/source_v1alpha1_gitrepository.yaml

@@ -7,5 +7,5 @@ metadata:
 spec:
   interval: 1m
   url: https://github.com/stefanprodan/podinfo
-  branch: master
+  ref:
-  tag: "3.2.2"
+    branch: master

controllers/gitrepository_controller.go

@@ -21,16 +21,21 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/blang/semver"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -126,95 +131,146 @@ func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourcev1.SourceCondition, string, error) {
+	// set defaults: master branch, no tags fetching, max two commits
+	branch := "master"
+	tagMode := git.NoTags
+	depth := 2
+
 	// determine ref
-	refName := plumbing.NewBranchReferenceName("master")
+	refName := plumbing.NewBranchReferenceName(branch)
-	if repository.Spec.Branch != "" {
+	if repository.Spec.Reference != nil {
-		refName = plumbing.NewBranchReferenceName(repository.Spec.Branch)
+		if repository.Spec.Reference.Branch != "" {
-	}
+			branch = repository.Spec.Reference.Branch
-	if repository.Spec.Tag != "" {
+			refName = plumbing.NewBranchReferenceName(branch)
-		refName = plumbing.NewTagReferenceName(repository.Spec.Tag)
+		}
+		if repository.Spec.Reference.Commit != "" {
+			depth = 0
+		} else {
+			if repository.Spec.Reference.Tag != "" {
+				refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag)
+			}
+			if repository.Spec.Reference.SemVer != "" {
+				tagMode = git.AllTags
+			}
+		}
 	}
 
-	// create tmp dir
+	// create tmp dir for SSH known_hosts
-	dir, err := ioutil.TempDir("", repository.Name)
+	tmpSSH, err := ioutil.TempDir("", repository.Name)
 	if err != nil {
-		err = fmt.Errorf("tmp dir error %w", err)
+		err = fmt.Errorf("tmp dir error: %w", err)
 		return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
 	}
-	defer os.RemoveAll(dir)
+	defer os.RemoveAll(tmpSSH)
+
+	auth, err := r.auth(repository, tmpSSH)
+	if err != nil {
+		err = fmt.Errorf("auth error: %w", err)
+		return NotReadyCondition(sourcev1.AuthenticationFailedReason, err.Error()), "", err
+	}
+
+	// create tmp dir for the Git clone
+	tmpGit, err := ioutil.TempDir("", repository.Name)
+	if err != nil {
+		err = fmt.Errorf("tmp dir error: %w", err)
+		return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
+	}
+	defer os.RemoveAll(tmpGit)
 
 	// clone to tmp
-	repo, err := git.PlainClone(dir, false, &git.CloneOptions{
+	repo, err := git.PlainClone(tmpGit, false, &git.CloneOptions{
-		URL:           repository.Spec.URL,
+		URL:               repository.Spec.URL,
-		Depth:         2,
+		Auth:              auth,
-		ReferenceName: refName,
+		RemoteName:        "origin",
-		SingleBranch:  true,
+		ReferenceName:     refName,
-		Tags:          git.AllTags,
+		SingleBranch:      true,
+		NoCheckout:        false,
+		Depth:             depth,
+		RecurseSubmodules: 0,
+		Progress:          nil,
+		Tags:              tagMode,
 	})
 	if err != nil {
-		err = fmt.Errorf("git clone error %w", err)
+		err = fmt.Errorf("git clone error: %w", err)
 		return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
 	}
 
-	// checkout tag based on semver expression
+	// checkout commit or tag
-	if repository.Spec.SemVer != "" {
+	if repository.Spec.Reference != nil {
-		rng, err := semver.ParseRange(repository.Spec.SemVer)
+		if commit := repository.Spec.Reference.Commit; commit != "" {
-		if err != nil {
-			err = fmt.Errorf("semver parse range error %w", err)
-			return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
-		}
-
-		repoTags, err := repo.Tags()
-		if err != nil {
-			err = fmt.Errorf("git list tags error %w", err)
-			return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
-		}
-
-		tags := make(map[string]string)
-		_ = repoTags.ForEach(func(t *plumbing.Reference) error {
-			tags[t.Name().Short()] = t.Strings()[1]
-			return nil
-		})
-
-		svTags := make(map[string]string)
-		svers := []semver.Version{}
-		for tag, _ := range tags {
-			v, _ := semver.ParseTolerant(tag)
-			if rng(v) {
-				svers = append(svers, v)
-				svTags[v.String()] = tag
-			}
-		}
-
-		if len(svers) > 0 {
-			semver.Sort(svers)
-			v := svers[len(svers)-1]
-			t := svTags[v.String()]
-			commit := tags[t]
-
 			w, err := repo.Worktree()
 			if err != nil {
-				err = fmt.Errorf("git worktree error %w", err)
+				err = fmt.Errorf("git worktree error: %w", err)
 				return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
 			}
 
 			err = w.Checkout(&git.CheckoutOptions{
-				Hash: plumbing.NewHash(commit),
+				Hash:  plumbing.NewHash(commit),
+				Force: true,
 			})
 			if err != nil {
-				err = fmt.Errorf("git checkout error %w", err)
+				err = fmt.Errorf("git checkout %s for %s error: %w", commit, branch, err)
+				return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
+			}
+		} else if exp := repository.Spec.Reference.SemVer; exp != "" {
+			rng, err := semver.ParseRange(exp)
+			if err != nil {
+				err = fmt.Errorf("semver parse range error: %w", err)
+				return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
+			}
+
+			repoTags, err := repo.Tags()
+			if err != nil {
+				err = fmt.Errorf("git list tags error: %w", err)
+				return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
+			}
+
+			tags := make(map[string]string)
+			_ = repoTags.ForEach(func(t *plumbing.Reference) error {
+				tags[t.Name().Short()] = t.Strings()[1]
+				return nil
+			})
+
+			svTags := make(map[string]string)
+			svers := []semver.Version{}
+			for tag, _ := range tags {
+				v, _ := semver.ParseTolerant(tag)
+				if rng(v) {
+					svers = append(svers, v)
+					svTags[v.String()] = tag
+				}
+			}
+
+			if len(svers) > 0 {
+				semver.Sort(svers)
+				v := svers[len(svers)-1]
+				t := svTags[v.String()]
+				commit := tags[t]
+
+				w, err := repo.Worktree()
+				if err != nil {
+					err = fmt.Errorf("git worktree error: %w", err)
+					return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
+				}
+
+				err = w.Checkout(&git.CheckoutOptions{
+					Hash: plumbing.NewHash(commit),
+				})
+				if err != nil {
+					err = fmt.Errorf("git checkout error: %w", err)
+					return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
+				}
+			} else {
+				err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer)
 				return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
 			}
-		} else {
-			err = fmt.Errorf("no match found for semver %s", repository.Spec.SemVer)
-			return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
 		}
 	}
 
 	// read commit hash
 	ref, err := repo.Head()
 	if err != nil {
-		err = fmt.Errorf("git resolve HEAD error %w", err)
+		err = fmt.Errorf("git resolve HEAD error: %w", err)
 		return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
 	}
 
@@ -224,7 +280,7 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc
 	// create artifact dir
 	err = r.Storage.MkdirAll(artifact)
 	if err != nil {
-		err = fmt.Errorf("mkdir dir error %w", err)
+		err = fmt.Errorf("mkdir dir error: %w", err)
 		return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
 	}
 
@@ -237,13 +293,20 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc
 	defer unlock()
 
 	// archive artifact
-	err = r.Storage.Archive(artifact, dir, "")
+	err = r.Storage.Archive(artifact, tmpGit, "")
 	if err != nil {
-		err = fmt.Errorf("storage error %w", err)
+		err = fmt.Errorf("storage archive error: %w", err)
 		return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
 	}
 
-	message := fmt.Sprintf("Artifact is available at %s", artifact.Path)
+	// update latest symlink
+	err = r.Storage.Symlink(artifact, "latest.tar.gz")
+	if err != nil {
+		err = fmt.Errorf("storage lock error: %w", err)
+		return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
+	}
+
+	message := fmt.Sprintf("Artifact is available at: %s", artifact.Path)
 	return ReadyCondition(sourcev1.GitOperationSucceedReason, message), artifact.URL, nil
 }
 
@@ -283,3 +346,74 @@ func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) {
 		}
 	}
 }
+
+func (r *GitRepositoryReconciler) auth(repository sourcev1.GitRepository, tmp string) (transport.AuthMethod, error) {
+	if repository.Spec.SecretRef == nil {
+		return nil, nil
+	}
+
+	name := types.NamespacedName{
+		Namespace: repository.GetNamespace(),
+		Name:      repository.Spec.SecretRef.Name,
+	}
+
+	var secret corev1.Secret
+	err := r.Client.Get(context.TODO(), name, &secret)
+	if err != nil {
+		return nil, err
+	}
+
+	credentials := secret.Data
+
+	// HTTP auth
+	if strings.HasPrefix(repository.Spec.URL, "http") {
+		auth := &http.BasicAuth{}
+		if username, ok := credentials["username"]; ok {
+			auth.Username = string(username)
+		}
+		if password, ok := credentials["password"]; ok {
+			auth.Password = string(password)
+		}
+
+		if auth.Username == "" || auth.Password == "" {
+			return nil, fmt.Errorf("invalid '%s' secret data: required fields username and password",
+				repository.Spec.SecretRef.Name)
+		}
+
+		return auth, nil
+	}
+
+	// SSH auth
+	if strings.HasPrefix(repository.Spec.URL, "ssh") {
+		var privateKey []byte
+		if identity, ok := credentials["identity"]; ok {
+			privateKey = identity
+		} else {
+			return nil, fmt.Errorf("invalid '%s' secret data: required field identity", repository.Spec.SecretRef.Name)
+		}
+
+		pk, err := ssh.NewPublicKeys("git", privateKey, "")
+		if err != nil {
+			return nil, err
+		}
+
+		known_hosts := filepath.Join(tmp, "known_hosts")
+		if kh, ok := credentials["known_hosts"]; ok {
+			if err := ioutil.WriteFile(filepath.Join(tmp, "known_hosts"), kh, 0644); err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, fmt.Errorf("invalid '%s' secret data: required field known_hosts", repository.Spec.SecretRef.Name)
+		}
+
+		callback, err := ssh.NewKnownHostsCallback(known_hosts)
+		if err != nil {
+			return nil, err
+		}
+		pk.HostKeyCallback = callback
+
+		return pk, nil
+	}
+
+	return nil, nil
+}

controllers/storage.go

@@ -95,7 +95,7 @@ func (s *Storage) RemoveAllButCurrent(artifact Artifact) error {
 	dir := filepath.Dir(artifact.Path)
 	errors := []string{}
 	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path != artifact.Path && !info.IsDir() {
+		if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
 			if err := os.Remove(path); err != nil {
 				errors = append(errors, info.Name())
 			}
@@ -149,11 +149,33 @@ func (s *Storage) WriteFile(artifact Artifact, data []byte) error {
 	return ioutil.WriteFile(artifact.Path, data, 0644)
 }
 
+// Symlink creates or updates a symbolic link for the given artifact
+func (s *Storage) Symlink(artifact Artifact, linkName string) error {
+	dir := filepath.Dir(artifact.Path)
+	link := filepath.Join(dir, linkName)
+	tmpLink := link + ".tmp"
+
+	if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if err := os.Symlink(artifact.Path, tmpLink); err != nil {
+		return err
+	}
+
+	if err := os.Rename(tmpLink, link); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Checksum returns the SHA1 checksum for the given bytes as a string
 func (s *Storage) Checksum(b []byte) string {
 	return fmt.Sprintf("%x", sha1.Sum(b))
 }
 
+// Lock creates a file lock for the given artifact
 func (s *Storage) Lock(artifact Artifact) (unlock func(), err error) {
 	lockFile := artifact.Path + ".lock"
 	mutex := lockedfile.MutexAt(lockFile)