fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6 from fluxcd/git-auth 

Implement Git authentication
This commit is contained in:
Stefan Prodan 2020-04-12 17:30:07 +03:00 committed by GitHub
parent 686fdd49dc 41d36f54ae
commit fa6bccbc72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 303 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/v1alpha1/condition_types.go
View File

@ -43,4 +43,8 @@ const (
// URLInvalidReason represents the fact that a given source has an invalid URL.
URLInvalidReason string = "URLInvalid"
// AuthenticationFailedReason represents the fact that a given secret doesn't
// have the required fields or the provided credentials don't match.
AuthenticationFailedReason string = "AuthenticationFailed"
)

20
api/v1alpha1/gitrepository_types.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package v1alpha1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -27,11 +28,24 @@ type GitRepositorySpec struct {
// +required
URL string `json:"url"`
// The secret name containing the Git credentials.
// For HTTPS repositories the secret must contain username and password fields.
// For SSH repositories the secret must contain identity, identity.pub and known_hosts fields.
// +optional
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
// The interval at which to check for repository updates.
// +required
Interval metav1.Duration `json:"interval"`
// The git branch to checkout, defaults to ('master').
// The git reference to checkout and monitor for changes, defaults to master branch.
// +optional
Reference *GitRepositoryRef `json:"ref,omitempty"`
}
// GitRepositoryRef defines the git ref used for pull and checkout operations
type GitRepositoryRef struct {
// The git branch to checkout, defaults to master.
// +optional
Branch string `json:"branch"`
@ -42,6 +56,10 @@ type GitRepositorySpec struct {
// The git tag semver expression, takes precedence over tag.
// +optional
SemVer string `json:"semver"`
// The git commit sha to checkout, if specified tag filters will be ignored.
// +optional
Commit string `json:"commit"`
}
// GitRepositoryStatus defines the observed state of GitRepository

28
api/v1alpha1/zz_generated.deepcopy.go
View File

@ -21,6 +21,7 @@ limitations under the License.
package v1alpha1
import (
"k8s.io/api/core/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -29,7 +30,7 @@ func (in *GitRepository) DeepCopyInto(out *GitRepository) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
@ -83,10 +84,35 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GitRepositoryRef) DeepCopyInto(out *GitRepositoryRef) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositoryRef.
func (in *GitRepositoryRef) DeepCopy() *GitRepositoryRef {
if in == nil {
return nil
}
out := new(GitRepositoryRef)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GitRepositorySpec) DeepCopyInto(out *GitRepositorySpec) {
*out = *in
if in.SecretRef != nil {
in, out := &in.SecretRef, &out.SecretRef
*out = new(v1.LocalObjectReference)
**out = **in
}
out.Interval = in.Interval
if in.Reference != nil {
in, out := &in.Reference, &out.Reference
*out = new(GitRepositoryRef)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositorySpec.

39
config/crd/bases/source.fluxcd.io_gitrepositories.yaml
View File

@ -49,18 +49,39 @@ spec:
spec:
description: GitRepositorySpec defines the desired state of GitRepository
properties:
branch:
description: The git branch to checkout, defaults to ('master').
type: string
interval:
description: The interval at which to check for repository updates.
type: string
semver:
description: The git tag semver expression, takes precedence over tag.
type: string
tag:
description: The git tag to checkout, takes precedence over branch.
type: string
ref:
description: The git reference to checkout and monitor for changes,
defaults to master branch.
properties:
branch:
description: The git branch to checkout, defaults to master.
type: string
commit:
description: The git commit sha to checkout, if specified tag filters
will be ignored.
type: string
semver:
description: The git tag semver expression, takes precedence over
tag.
type: string
tag:
description: The git tag to checkout, takes precedence over branch.
type: string
type: object
secretRef:
description: The secret name containing the Git credentials. For HTTPS
repositories the secret must contain username and password fields.
For SSH repositories the secret must contain identity, identity.pub
and known_hosts fields.
properties:
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
type: object
url:
description: The repository URL, can be a HTTP or SSH address.
pattern: ^(http|https|ssh)://

4
config/samples/source_v1alpha1_gitrepository.yaml
View File

@ -7,5 +7,5 @@ metadata:
spec:
interval: 1m
url: https://github.com/stefanprodan/podinfo
branch: master
tag: "3.2.2"
ref:
branch: master

262
controllers/gitrepository_controller.go
View File

@ -21,16 +21,21 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/blang/semver"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
@ -126,95 +131,146 @@ func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error {
}
func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourcev1.SourceCondition, string, error) {
// set defaults: master branch, no tags fetching, max two commits
branch := "master"
tagMode := git.NoTags
depth := 2
// determine ref
refName := plumbing.NewBranchReferenceName("master")
if repository.Spec.Branch != "" {
refName = plumbing.NewBranchReferenceName(repository.Spec.Branch)
}
if repository.Spec.Tag != "" {
refName = plumbing.NewTagReferenceName(repository.Spec.Tag)
refName := plumbing.NewBranchReferenceName(branch)
if repository.Spec.Reference != nil {
if repository.Spec.Reference.Branch != "" {
branch = repository.Spec.Reference.Branch
refName = plumbing.NewBranchReferenceName(branch)
}
if repository.Spec.Reference.Commit != "" {
depth = 0
} else {
if repository.Spec.Reference.Tag != "" {
refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag)
}
if repository.Spec.Reference.SemVer != "" {
tagMode = git.AllTags
}
}
}
// create tmp dir
dir, err := ioutil.TempDir("", repository.Name)
// create tmp dir for SSH known_hosts
tmpSSH, err := ioutil.TempDir("", repository.Name)
if err != nil {
err = fmt.Errorf("tmp dir error %w", err)
err = fmt.Errorf("tmp dir error: %w", err)
return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
defer os.RemoveAll(dir)
defer os.RemoveAll(tmpSSH)
auth, err := r.auth(repository, tmpSSH)
if err != nil {
err = fmt.Errorf("auth error: %w", err)
return NotReadyCondition(sourcev1.AuthenticationFailedReason, err.Error()), "", err
}
// create tmp dir for the Git clone
tmpGit, err := ioutil.TempDir("", repository.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
defer os.RemoveAll(tmpGit)
// clone to tmp
repo, err := git.PlainClone(dir, false, &git.CloneOptions{
URL: repository.Spec.URL,
Depth: 2,
ReferenceName: refName,
SingleBranch: true,
Tags: git.AllTags,
repo, err := git.PlainClone(tmpGit, false, &git.CloneOptions{
URL: repository.Spec.URL,
Auth: auth,
RemoteName: "origin",
ReferenceName: refName,
SingleBranch: true,
NoCheckout: false,
Depth: depth,
RecurseSubmodules: 0,
Progress: nil,
Tags: tagMode,
})
if err != nil {
err = fmt.Errorf("git clone error %w", err)
err = fmt.Errorf("git clone error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
// checkout tag based on semver expression
if repository.Spec.SemVer != "" {
rng, err := semver.ParseRange(repository.Spec.SemVer)
if err != nil {
err = fmt.Errorf("semver parse range error %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
repoTags, err := repo.Tags()
if err != nil {
err = fmt.Errorf("git list tags error %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
tags := make(map[string]string)
_ = repoTags.ForEach(func(t *plumbing.Reference) error {
tags[t.Name().Short()] = t.Strings()[1]
return nil
})
svTags := make(map[string]string)
svers := []semver.Version{}
for tag, _ := range tags {
v, _ := semver.ParseTolerant(tag)
if rng(v) {
svers = append(svers, v)
svTags[v.String()] = tag
}
}
if len(svers) > 0 {
semver.Sort(svers)
v := svers[len(svers)-1]
t := svTags[v.String()]
commit := tags[t]
// checkout commit or tag
if repository.Spec.Reference != nil {
if commit := repository.Spec.Reference.Commit; commit != "" {
w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error %w", err)
err = fmt.Errorf("git worktree error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
Hash: plumbing.NewHash(commit),
Force: true,
})
if err != nil {
err = fmt.Errorf("git checkout error %w", err)
err = fmt.Errorf("git checkout %s for %s error: %w", commit, branch, err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
} else if exp := repository.Spec.Reference.SemVer; exp != "" {
rng, err := semver.ParseRange(exp)
if err != nil {
err = fmt.Errorf("semver parse range error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
repoTags, err := repo.Tags()
if err != nil {
err = fmt.Errorf("git list tags error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
tags := make(map[string]string)
_ = repoTags.ForEach(func(t *plumbing.Reference) error {
tags[t.Name().Short()] = t.Strings()[1]
return nil
})
svTags := make(map[string]string)
svers := []semver.Version{}
for tag, _ := range tags {
v, _ := semver.ParseTolerant(tag)
if rng(v) {
svers = append(svers, v)
svTags[v.String()] = tag
}
}
if len(svers) > 0 {
semver.Sort(svers)
v := svers[len(svers)-1]
t := svTags[v.String()]
commit := tags[t]
w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
})
if err != nil {
err = fmt.Errorf("git checkout error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
} else {
err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
} else {
err = fmt.Errorf("no match found for semver %s", repository.Spec.SemVer)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
}
// read commit hash
ref, err := repo.Head()
if err != nil {
err = fmt.Errorf("git resolve HEAD error %w", err)
err = fmt.Errorf("git resolve HEAD error: %w", err)
return NotReadyCondition(sourcev1.GitOperationFailedReason, err.Error()), "", err
}
@ -224,7 +280,7 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc
// create artifact dir
err = r.Storage.MkdirAll(artifact)
if err != nil {
err = fmt.Errorf("mkdir dir error %w", err)
err = fmt.Errorf("mkdir dir error: %w", err)
return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
@ -237,13 +293,20 @@ func (r *GitRepositoryReconciler) sync(repository sourcev1.GitRepository) (sourc
defer unlock()
// archive artifact
err = r.Storage.Archive(artifact, dir, "")
err = r.Storage.Archive(artifact, tmpGit, "")
if err != nil {
err = fmt.Errorf("storage error %w", err)
err = fmt.Errorf("storage archive error: %w", err)
return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
message := fmt.Sprintf("Artifact is available at %s", artifact.Path)
// update latest symlink
err = r.Storage.Symlink(artifact, "latest.tar.gz")
if err != nil {
err = fmt.Errorf("storage lock error: %w", err)
return NotReadyCondition(sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
message := fmt.Sprintf("Artifact is available at: %s", artifact.Path)
return ReadyCondition(sourcev1.GitOperationSucceedReason, message), artifact.URL, nil
}
@ -283,3 +346,74 @@ func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) {
}
}
}
func (r *GitRepositoryReconciler) auth(repository sourcev1.GitRepository, tmp string) (transport.AuthMethod, error) {
if repository.Spec.SecretRef == nil {
return nil, nil
}
name := types.NamespacedName{
Namespace: repository.GetNamespace(),
Name: repository.Spec.SecretRef.Name,
}
var secret corev1.Secret
err := r.Client.Get(context.TODO(), name, &secret)
if err != nil {
return nil, err
}
credentials := secret.Data
// HTTP auth
if strings.HasPrefix(repository.Spec.URL, "http") {
auth := &http.BasicAuth{}
if username, ok := credentials["username"]; ok {
auth.Username = string(username)
}
if password, ok := credentials["password"]; ok {
auth.Password = string(password)
}
if auth.Username == "" || auth.Password == "" {
return nil, fmt.Errorf("invalid '%s' secret data: required fields username and password",
repository.Spec.SecretRef.Name)
}
return auth, nil
}
// SSH auth
if strings.HasPrefix(repository.Spec.URL, "ssh") {
var privateKey []byte
if identity, ok := credentials["identity"]; ok {
privateKey = identity
} else {
return nil, fmt.Errorf("invalid '%s' secret data: required field identity", repository.Spec.SecretRef.Name)
}
pk, err := ssh.NewPublicKeys("git", privateKey, "")
if err != nil {
return nil, err
}
known_hosts := filepath.Join(tmp, "known_hosts")
if kh, ok := credentials["known_hosts"]; ok {
if err := ioutil.WriteFile(filepath.Join(tmp, "known_hosts"), kh, 0644); err != nil {
return nil, err
}
} else {
return nil, fmt.Errorf("invalid '%s' secret data: required field known_hosts", repository.Spec.SecretRef.Name)
}
callback, err := ssh.NewKnownHostsCallback(known_hosts)
if err != nil {
return nil, err
}
pk.HostKeyCallback = callback
return pk, nil
}
return nil, nil
}

24
controllers/storage.go
View File

@ -95,7 +95,7 @@ func (s *Storage) RemoveAllButCurrent(artifact Artifact) error {
dir := filepath.Dir(artifact.Path)
errors := []string{}
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if path != artifact.Path && !info.IsDir() {
if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if err := os.Remove(path); err != nil {
errors = append(errors, info.Name())
}
@ -149,11 +149,33 @@ func (s *Storage) WriteFile(artifact Artifact, data []byte) error {
return ioutil.WriteFile(artifact.Path, data, 0644)
}
// Symlink creates or updates a symbolic link for the given artifact
func (s *Storage) Symlink(artifact Artifact, linkName string) error {
dir := filepath.Dir(artifact.Path)
link := filepath.Join(dir, linkName)
tmpLink := link + ".tmp"
if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) {
return err
}
if err := os.Symlink(artifact.Path, tmpLink); err != nil {
return err
}
if err := os.Rename(tmpLink, link); err != nil {
return err
}
return nil
}
// Checksum returns the SHA1 checksum for the given bytes as a string
func (s *Storage) Checksum(b []byte) string {
return fmt.Sprintf("%x", sha1.Sum(b))
}
// Lock creates a file lock for the given artifact
func (s *Storage) Lock(artifact Artifact) (unlock func(), err error) {
lockFile := artifact.Path + ".lock"
mutex := lockedfile.MutexAt(lockFile)