Implement repository artifacts local storage
This commit is contained in:
stefanprodan
parent
3efd54efd1
commit
037db0bc02
|
|
@ -19,12 +19,15 @@ package controllers
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
"github.com/go-logr/logr"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -39,8 +42,9 @@ import (
|
|||
// GitRepositoryReconciler reconciles a GitRepository object
|
||||
type GitRepositoryReconciler struct {
|
||||
client.Client
|
||||
Log logr.Logger
|
||||
Scheme *runtime.Scheme
|
||||
Log logr.Logger
|
||||
Scheme *runtime.Scheme
|
||||
StoragePath string
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=sourcer.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete
|
||||
|
|
@ -60,7 +64,10 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
|
|||
result := ctrl.Result{RequeueAfter: repo.Spec.Interval.Duration}
|
||||
|
||||
// set initial status
|
||||
if len(repo.Status.Conditions) == 0 {
|
||||
if r.shouldResetStatus(repo) {
|
||||
log.Info("Initialising repository")
|
||||
repo.Status.Artifacts = ""
|
||||
repo.Status.LastUpdateTime = nil
|
||||
repo.Status.Conditions = []sourcerv1.RepositoryCondition{
|
||||
{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
|
|
@ -74,10 +81,16 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
|
|||
}
|
||||
|
||||
// try git clone
|
||||
readyCondition, err := r.sync(repo.Spec)
|
||||
readyCondition, artifacts, err := r.sync(repo)
|
||||
if err != nil {
|
||||
log.Info("Repository sync failed", "error", err.Error())
|
||||
} else {
|
||||
// update artifacts if commit hash changed
|
||||
if repo.Status.Artifacts != artifacts {
|
||||
timeNew := metav1.Now()
|
||||
repo.Status.LastUpdateTime = &timeNew
|
||||
repo.Status.Artifacts = artifacts
|
||||
}
|
||||
log.Info("Repository sync succeeded", "msg", readyCondition.Message)
|
||||
}
|
||||
|
||||
|
|
@ -98,29 +111,50 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
|
|||
func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&sourcerv1.GitRepository{}).
|
||||
WithEventFilter(RepositoryChangePredicate{}).
|
||||
WithEventFilter(predicate.Funcs{
|
||||
DeleteFunc: func(e event.DeleteEvent) bool {
|
||||
//TODO: cleanup
|
||||
// delete artifacts
|
||||
repoDir := filepath.Join(r.StoragePath,
|
||||
fmt.Sprintf("repositories/%s-%s", e.Meta.GetName(), e.Meta.GetNamespace()))
|
||||
if err := os.RemoveAll(repoDir); err != nil {
|
||||
r.Log.Error(err, "unable to delete artifacts",
|
||||
"gitrepository", fmt.Sprintf("%s/%s", e.Meta.GetNamespace(), e.Meta.GetName()))
|
||||
} else {
|
||||
r.Log.Info("Repository artifacts deleted",
|
||||
"gitrepository", fmt.Sprintf("%s/%s", e.Meta.GetNamespace(), e.Meta.GetName()))
|
||||
}
|
||||
return false
|
||||
},
|
||||
}).
|
||||
WithEventFilter(RepositoryChangePredicate{}).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (sourcerv1.RepositoryCondition, error) {
|
||||
func (r *GitRepositoryReconciler) sync(gr sourcerv1.GitRepository) (sourcerv1.RepositoryCondition, string, error) {
|
||||
// determine ref
|
||||
refName := plumbing.NewBranchReferenceName("master")
|
||||
if spec.Branch != "" {
|
||||
refName = plumbing.NewBranchReferenceName(spec.Branch)
|
||||
if gr.Spec.Branch != "" {
|
||||
refName = plumbing.NewBranchReferenceName(gr.Spec.Branch)
|
||||
}
|
||||
if spec.Tag != "" {
|
||||
refName = plumbing.NewTagReferenceName(spec.Tag)
|
||||
if gr.Spec.Tag != "" {
|
||||
refName = plumbing.NewTagReferenceName(gr.Spec.Tag)
|
||||
}
|
||||
|
||||
// clone
|
||||
repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
|
||||
URL: spec.Url,
|
||||
dir, err := ioutil.TempDir("", gr.Name)
|
||||
if err != nil {
|
||||
ex := fmt.Errorf("tmp dir error %w", err)
|
||||
return sourcerv1.RepositoryCondition{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
Status: corev1.ConditionFalse,
|
||||
Reason: "ExecFailed",
|
||||
Message: ex.Error(),
|
||||
}, "", ex
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
// clone to tmp
|
||||
repo, err := git.PlainClone(dir, false, &git.CloneOptions{
|
||||
URL: gr.Spec.Url,
|
||||
Depth: 2,
|
||||
ReferenceName: refName,
|
||||
SingleBranch: true,
|
||||
|
|
@ -132,7 +166,7 @@ func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (source
|
|||
Status: corev1.ConditionFalse,
|
||||
Reason: "GitCloneFailed",
|
||||
Message: ex.Error(),
|
||||
}, ex
|
||||
}, "", ex
|
||||
}
|
||||
|
||||
// read commit hash
|
||||
|
|
@ -142,15 +176,62 @@ func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (source
|
|||
return sourcerv1.RepositoryCondition{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
Status: corev1.ConditionFalse,
|
||||
Reason: "GitCommandFailed",
|
||||
Reason: "GitHeadFailed",
|
||||
Message: ex.Error(),
|
||||
}, ex
|
||||
}, "", ex
|
||||
}
|
||||
|
||||
// create artifacts dir
|
||||
repoDir := fmt.Sprintf("repositories/%s-%s", gr.Name, gr.Namespace)
|
||||
storage := filepath.Join(r.StoragePath, repoDir)
|
||||
err = os.MkdirAll(storage, 0777)
|
||||
if err != nil {
|
||||
ex := fmt.Errorf("mkdir dir error %w", err)
|
||||
return sourcerv1.RepositoryCondition{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
Status: corev1.ConditionFalse,
|
||||
Reason: "ExecFailed",
|
||||
Message: ex.Error(),
|
||||
}, "", ex
|
||||
}
|
||||
|
||||
// store artifacts
|
||||
artifacts := filepath.Join(storage, fmt.Sprintf("%s.tar.gz", ref.Hash().String()))
|
||||
excludes := "--exclude=\\*.{jpg,gif,png,wmv,flv,tar.gz,zip} --exclude .git"
|
||||
command := exec.Command("/bin/sh", "-c",
|
||||
fmt.Sprintf("cd %s && tar -c %s -f - . | gzip > %s", dir, excludes, artifacts))
|
||||
err = command.Run()
|
||||
if err != nil {
|
||||
ex := fmt.Errorf("tar %s error %w", artifacts, err)
|
||||
return sourcerv1.RepositoryCondition{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
Status: corev1.ConditionFalse,
|
||||
Reason: "ExecFailed",
|
||||
Message: ex.Error(),
|
||||
}, "", ex
|
||||
}
|
||||
|
||||
output := fmt.Sprintf("/repositories/%s-%s/%s.tar.gz", gr.Name, gr.Namespace, ref.Hash().String())
|
||||
return sourcerv1.RepositoryCondition{
|
||||
Type: sourcerv1.RepositoryConditionReady,
|
||||
Status: corev1.ConditionTrue,
|
||||
Reason: "GitCloneSucceed",
|
||||
Message: fmt.Sprintf("commit hash %s", ref.Hash().String()),
|
||||
}, nil
|
||||
Message: fmt.Sprintf("Fetched artifacts are available at %s", artifacts),
|
||||
}, output, nil
|
||||
}
|
||||
|
||||
func (r *GitRepositoryReconciler) shouldResetStatus(gr sourcerv1.GitRepository) bool {
|
||||
resetStatus := false
|
||||
if gr.Status.Artifacts != "" {
|
||||
if _, err := os.Stat(filepath.Join(r.StoragePath, gr.Status.Artifacts)); err != nil {
|
||||
resetStatus = true
|
||||
}
|
||||
}
|
||||
|
||||
// set initial status
|
||||
if len(gr.Status.Conditions) == 0 || resetStatus {
|
||||
resetStatus = true
|
||||
}
|
||||
|
||||
return resetStatus
|
||||
}
|
||||
|
|
|
|||
7
go.sum
7
go.sum
|
|
@ -15,8 +15,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
|||
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
|
||||
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
|
||||
|
|
@ -107,11 +109,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+
|
|||
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
|
||||
github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
|
||||
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
|
||||
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
|
||||
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
|
||||
github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
|
||||
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
|
||||
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
|
||||
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
|
||||
github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
|
||||
github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
|
||||
|
|
@ -125,6 +129,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
|
|||
github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
|
||||
github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
|
||||
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
|
||||
github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
|
||||
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
|
||||
github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
|
||||
github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
|
||||
|
|
@ -134,6 +139,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
|
|||
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
|
||||
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
|
||||
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
|
||||
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
|
||||
|
|
@ -220,6 +226,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
|
|||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
|
||||
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
|
|
|
|||
32
main.go
32
main.go
|
|
@ -18,7 +18,10 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/go-logr/logr"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
|
|
@ -46,10 +49,15 @@ func init() {
|
|||
func main() {
|
||||
var metricsAddr string
|
||||
var enableLeaderElection bool
|
||||
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
|
||||
var storagePath string
|
||||
var storageAddr string
|
||||
flag.StringVar(&metricsAddr, "metrics-addr", ":9090", "The address the metric endpoint binds to.")
|
||||
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
|
||||
"Enable leader election for controller manager. "+
|
||||
"Enabling this will ensure there is only one active controller manager.")
|
||||
flag.StringVar(&storagePath, "storage-path", "", "The local storage path.")
|
||||
flag.StringVar(&storageAddr, "storage-addr", ":8080", "The address the static file server binds to.")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
|
||||
|
|
@ -66,10 +74,17 @@ func main() {
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
if storagePath == "" {
|
||||
p, _ := os.Getwd()
|
||||
storagePath = filepath.Join(p, "bin")
|
||||
}
|
||||
go startFileServer(storagePath, storageAddr, setupLog)
|
||||
|
||||
if err = (&controllers.GitRepositoryReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("GitRepository"),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Client: mgr.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("GitRepository"),
|
||||
Scheme: mgr.GetScheme(),
|
||||
StoragePath: storagePath,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "GitRepository")
|
||||
os.Exit(1)
|
||||
|
|
@ -90,3 +105,12 @@ func main() {
|
|||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func startFileServer(path string, address string, l logr.Logger) {
|
||||
fs := http.FileServer(http.Dir(path))
|
||||
http.Handle("/", fs)
|
||||
err := http.ListenAndServe(address, nil)
|
||||
if err != nil {
|
||||
l.Error(err, "file server error")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue