From 037db0bc020d13f37b2a90bad1bd853701964634 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Mon, 6 Apr 2020 19:02:46 +0300 Subject: [PATCH] Implement repository artifacts local storage --- controllers/gitrepository_controller.go | 121 ++++++++++++++++++++---- go.sum | 7 ++ main.go | 32 ++++++- 3 files changed, 136 insertions(+), 24 deletions(-) diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 538c26fc..280b1193 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -19,12 +19,15 @@ package controllers import ( "context" "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" "sigs.k8s.io/controller-runtime/pkg/event" "time" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/storage/memory" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,8 +42,9 @@ import ( // GitRepositoryReconciler reconciles a GitRepository object type GitRepositoryReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + StoragePath string } // +kubebuilder:rbac:groups=sourcer.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete @@ -60,7 +64,10 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro result := ctrl.Result{RequeueAfter: repo.Spec.Interval.Duration} // set initial status - if len(repo.Status.Conditions) == 0 { + if r.shouldResetStatus(repo) { + log.Info("Initialising repository") + repo.Status.Artifacts = "" + repo.Status.LastUpdateTime = nil repo.Status.Conditions = []sourcerv1.RepositoryCondition{ { Type: sourcerv1.RepositoryConditionReady, @@ -74,10 +81,16 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro } // try git clone - readyCondition, err := r.sync(repo.Spec) + readyCondition, artifacts, err := r.sync(repo) if err != nil { log.Info("Repository sync failed", "error", err.Error()) } else { + // update artifacts if commit hash changed + if repo.Status.Artifacts != artifacts { + timeNew := metav1.Now() + repo.Status.LastUpdateTime = &timeNew + repo.Status.Artifacts = artifacts + } log.Info("Repository sync succeeded", "msg", readyCondition.Message) } @@ -98,29 +111,50 @@ func (r *GitRepositoryReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&sourcerv1.GitRepository{}). + WithEventFilter(RepositoryChangePredicate{}). WithEventFilter(predicate.Funcs{ DeleteFunc: func(e event.DeleteEvent) bool { - //TODO: cleanup + // delete artifacts + repoDir := filepath.Join(r.StoragePath, + fmt.Sprintf("repositories/%s-%s", e.Meta.GetName(), e.Meta.GetNamespace())) + if err := os.RemoveAll(repoDir); err != nil { + r.Log.Error(err, "unable to delete artifacts", + "gitrepository", fmt.Sprintf("%s/%s", e.Meta.GetNamespace(), e.Meta.GetName())) + } else { + r.Log.Info("Repository artifacts deleted", + "gitrepository", fmt.Sprintf("%s/%s", e.Meta.GetNamespace(), e.Meta.GetName())) + } return false }, }). - WithEventFilter(RepositoryChangePredicate{}). Complete(r) } -func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (sourcerv1.RepositoryCondition, error) { +func (r *GitRepositoryReconciler) sync(gr sourcerv1.GitRepository) (sourcerv1.RepositoryCondition, string, error) { // determine ref refName := plumbing.NewBranchReferenceName("master") - if spec.Branch != "" { - refName = plumbing.NewBranchReferenceName(spec.Branch) + if gr.Spec.Branch != "" { + refName = plumbing.NewBranchReferenceName(gr.Spec.Branch) } - if spec.Tag != "" { - refName = plumbing.NewTagReferenceName(spec.Tag) + if gr.Spec.Tag != "" { + refName = plumbing.NewTagReferenceName(gr.Spec.Tag) } - // clone - repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ - URL: spec.Url, + dir, err := ioutil.TempDir("", gr.Name) + if err != nil { + ex := fmt.Errorf("tmp dir error %w", err) + return sourcerv1.RepositoryCondition{ + Type: sourcerv1.RepositoryConditionReady, + Status: corev1.ConditionFalse, + Reason: "ExecFailed", + Message: ex.Error(), + }, "", ex + } + defer os.RemoveAll(dir) + + // clone to tmp + repo, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: gr.Spec.Url, Depth: 2, ReferenceName: refName, SingleBranch: true, @@ -132,7 +166,7 @@ func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (source Status: corev1.ConditionFalse, Reason: "GitCloneFailed", Message: ex.Error(), - }, ex + }, "", ex } // read commit hash @@ -142,15 +176,62 @@ func (r *GitRepositoryReconciler) sync(spec sourcerv1.GitRepositorySpec) (source return sourcerv1.RepositoryCondition{ Type: sourcerv1.RepositoryConditionReady, Status: corev1.ConditionFalse, - Reason: "GitCommandFailed", + Reason: "GitHeadFailed", Message: ex.Error(), - }, ex + }, "", ex } + // create artifacts dir + repoDir := fmt.Sprintf("repositories/%s-%s", gr.Name, gr.Namespace) + storage := filepath.Join(r.StoragePath, repoDir) + err = os.MkdirAll(storage, 0777) + if err != nil { + ex := fmt.Errorf("mkdir dir error %w", err) + return sourcerv1.RepositoryCondition{ + Type: sourcerv1.RepositoryConditionReady, + Status: corev1.ConditionFalse, + Reason: "ExecFailed", + Message: ex.Error(), + }, "", ex + } + + // store artifacts + artifacts := filepath.Join(storage, fmt.Sprintf("%s.tar.gz", ref.Hash().String())) + excludes := "--exclude=\\*.{jpg,gif,png,wmv,flv,tar.gz,zip} --exclude .git" + command := exec.Command("/bin/sh", "-c", + fmt.Sprintf("cd %s && tar -c %s -f - . | gzip > %s", dir, excludes, artifacts)) + err = command.Run() + if err != nil { + ex := fmt.Errorf("tar %s error %w", artifacts, err) + return sourcerv1.RepositoryCondition{ + Type: sourcerv1.RepositoryConditionReady, + Status: corev1.ConditionFalse, + Reason: "ExecFailed", + Message: ex.Error(), + }, "", ex + } + + output := fmt.Sprintf("/repositories/%s-%s/%s.tar.gz", gr.Name, gr.Namespace, ref.Hash().String()) return sourcerv1.RepositoryCondition{ Type: sourcerv1.RepositoryConditionReady, Status: corev1.ConditionTrue, Reason: "GitCloneSucceed", - Message: fmt.Sprintf("commit hash %s", ref.Hash().String()), - }, nil + Message: fmt.Sprintf("Fetched artifacts are available at %s", artifacts), + }, output, nil +} + +func (r *GitRepositoryReconciler) shouldResetStatus(gr sourcerv1.GitRepository) bool { + resetStatus := false + if gr.Status.Artifacts != "" { + if _, err := os.Stat(filepath.Join(r.StoragePath, gr.Status.Artifacts)); err != nil { + resetStatus = true + } + } + + // set initial status + if len(gr.Status.Conditions) == 0 || resetStatus { + resetStatus = true + } + + return resetStatus } diff --git a/go.sum b/go.sum index a0270295..3aea78ed 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= @@ -107,11 +109,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -125,6 +129,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -134,6 +139,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -220,6 +226,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= diff --git a/main.go b/main.go index 5525bf26..206750ef 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,10 @@ package main import ( "flag" + "github.com/go-logr/logr" + "net/http" "os" + "path/filepath" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -46,10 +49,15 @@ func init() { func main() { var metricsAddr string var enableLeaderElection bool - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") + var storagePath string + var storageAddr string + flag.StringVar(&metricsAddr, "metrics-addr", ":9090", "The address the metric endpoint binds to.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&storagePath, "storage-path", "", "The local storage path.") + flag.StringVar(&storageAddr, "storage-addr", ":8080", "The address the static file server binds to.") + flag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -66,10 +74,17 @@ func main() { os.Exit(1) } + if storagePath == "" { + p, _ := os.Getwd() + storagePath = filepath.Join(p, "bin") + } + go startFileServer(storagePath, storageAddr, setupLog) + if err = (&controllers.GitRepositoryReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GitRepository"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GitRepository"), + Scheme: mgr.GetScheme(), + StoragePath: storagePath, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GitRepository") os.Exit(1) @@ -90,3 +105,12 @@ func main() { os.Exit(1) } } + +func startFileServer(path string, address string, l logr.Logger) { + fs := http.FileServer(http.Dir(path)) + http.Handle("/", fs) + err := http.ListenAndServe(address, nil) + if err != nil { + l.Error(err, "file server error") + } +}