From a138a2b19bcdf69670c1bf4297d1181da42c13a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Casta=C3=B1o=20Arteaga?= Date: Tue, 20 Sep 2022 17:26:48 +0200 Subject: [PATCH] Add experimental support for Tekton git versioning (#2337) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergio Castaño Arteaga Signed-off-by: Cintia Sanchez Garcia Co-authored-by: Sergio Castaño Arteaga Co-authored-by: Cintia Sanchez Garcia --- cmd/ah/lint.go | 16 +- internal/hub/repo.go | 12 ++ internal/repo/manager.go | 11 ++ internal/tracker/source/tekton/tekton.go | 187 +++++++++++++++--- internal/tracker/source/tekton/tekton_test.go | 8 + .../controlPanel/repositories/Modal.tsx | 66 ++++++- web/src/types.ts | 6 + 7 files changed, 273 insertions(+), 33 deletions(-) diff --git a/cmd/ah/lint.go b/cmd/ah/lint.go index 16d34162..9fd0b762 100644 --- a/cmd/ah/lint.go +++ b/cmd/ah/lint.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "errors" "fmt" "io" @@ -374,6 +375,7 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport { repository := &hub.Repository{ Kind: kind, URL: "https://github.com/user/repo/path", + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), } // Read catalog path to get available packages @@ -399,7 +401,8 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport { if !p.IsDir() { continue } - if _, err := semver.NewVersion(v.Name()); err != nil { + sv, err := semver.NewVersion(v.Name()) + if err != nil { continue } @@ -416,7 +419,16 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport { e.result = multierror.Append(e.result, err) } else { // Prepare package version - e.pkg, err = tekton.PreparePackage(repository, manifest, manifestRaw, basePath, pkgPath) + e.pkg, err = tekton.PreparePackage(&tekton.PreparePackageInput{ + R: repository, + Tag: "", + Manifest: manifest, + ManifestRaw: manifestRaw, + BasePath: basePath, + PkgName: pkgName, + PkgPath: pkgPath, + PkgVersion: sv.String(), + }) if err != nil { e.result = multierror.Append(e.result, err) } diff --git a/internal/hub/repo.go b/internal/hub/repo.go index f4b0d9ff..95c6b480 100644 --- a/internal/hub/repo.go +++ b/internal/hub/repo.go @@ -18,6 +18,12 @@ const ( RepositoryOCIPrefix = "oci://" ) +const ( + // Tekton catalog versioning kinds + TektonDirBasedVersioning = "directory" + TektonGitBasedVersioning = "git" +) + // ContainerImageData represents some data specific to repositories of the // container image kind. type ContainerImageData struct { @@ -30,6 +36,12 @@ type ContainerImageTag struct { Mutable bool `json:"mutable"` } +// TektonData represents some data specific to repositories of the Tekton tasks +// or pipelines kinds. +type TektonData struct { + Versioning string `json:"versioning"` // Options: directory or git +} + // RepositoryKind represents the kind of a given repository. type RepositoryKind int64 diff --git a/internal/repo/manager.go b/internal/repo/manager.go index a5ce7596..aed5bc9a 100644 --- a/internal/repo/manager.go +++ b/internal/repo/manager.go @@ -554,6 +554,17 @@ func (m *Manager) GetRemoteDigest(ctx context.Context, r *hub.Repository) (strin digest = desc.Digest.String() case GitRepoURLRE.MatchString(r.URL): + // Do not track repo's digest for Tekton repos using git based versioning + if r.Data != nil { + var data *hub.TektonData + if err := json.Unmarshal(r.Data, &data); err != nil { + return "", fmt.Errorf("invalid tekton repository data: %w", err) + } + if data.Versioning == hub.TektonGitBasedVersioning { + return "", nil + } + } + // Digest is obtained from the last commit in the repository matches := GitRepoURLRE.FindStringSubmatch(r.URL) repoBaseURL := matches[1] diff --git a/internal/tracker/source/tekton/tekton.go b/internal/tracker/source/tekton/tekton.go index ec1cf8b9..979c28ed 100644 --- a/internal/tracker/source/tekton/tekton.go +++ b/internal/tracker/source/tekton/tekton.go @@ -1,6 +1,8 @@ package tekton import ( + "crypto/sha256" + "encoding/json" "errors" "fmt" "os" @@ -15,6 +17,8 @@ import ( "github.com/artifacthub/hub/internal/tracker/source" "github.com/artifacthub/hub/internal/tracker/source/generic" "github.com/ghodss/yaml" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/hashicorp/go-multierror" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" ) @@ -84,6 +88,29 @@ func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource { // GetPackagesAvailable implements the TrackerSource interface. func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) { + // Get repository's Tekton specific data + if s.i.Repository.Data == nil { + return nil, errors.New("required repository data field not provided") + } + var data *hub.TektonData + if err := json.Unmarshal(s.i.Repository.Data, &data); err != nil { + return nil, fmt.Errorf("invalid tekton repository data: %w", err) + } + + // Process catalog based on the versioning option configured + switch data.Versioning { + case hub.TektonDirBasedVersioning: + return s.processDirBasedCatalog() + case hub.TektonGitBasedVersioning: + return s.processGitBasedCatalog() + default: + return nil, fmt.Errorf("invalid catalog versioning option: %s", data.Versioning) + } +} + +// processDirBasedCatalog returns the packages available in the catalog using +// the directory based versioning. +func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error) { packagesAvailable := make(map[string]*hub.Package) // Read catalog path to get available packages @@ -117,7 +144,8 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) if !p.IsDir() { continue } - if _, err := semver.NewVersion(v.Name()); err != nil { + sv, err := semver.NewVersion(v.Name()) + if err != nil { continue } @@ -130,7 +158,16 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) } // Prepare and store package version - p, err := PreparePackage(s.i.Repository, manifest, manifestRaw, s.i.BasePath, pkgPath) + p, err := PreparePackage(&PreparePackageInput{ + R: s.i.Repository, + Tag: "", + Manifest: manifest, + ManifestRaw: manifestRaw, + BasePath: s.i.BasePath, + PkgName: pkgName, + PkgPath: pkgPath, + PkgVersion: sv.String(), + }) if err != nil { s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, v.Name(), err)) continue @@ -142,6 +179,87 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) return packagesAvailable, nil } +// processGitBasedCatalog returns the packages available in the catalog using +// the git based versioning. +func (s *TrackerSource) processGitBasedCatalog() (map[string]*hub.Package, error) { + // Open git repository and get all tags + gr, err := git.PlainOpenWithOptions(s.i.BasePath, &git.PlainOpenOptions{ + DetectDotGit: true, + }) + if err != nil { + return nil, fmt.Errorf("error opening git repository: %w", err) + } + wt, err := gr.Worktree() + if err != nil { + return nil, fmt.Errorf("error getting worktree: %w", err) + } + tags, err := gr.Tags() + if err != nil { + return nil, fmt.Errorf("error reading tags references: %w", err) + } + + // Read packages available in the catalog for each tag/version + packagesAvailable := make(map[string]*hub.Package) + _ = tags.ForEach(func(tag *plumbing.Reference) error { + // Skip tags that cannot be parsed as ~valid semver + sv, err := semver.NewVersion(tag.Name().Short()) + if err != nil { + return nil + } + + // Checkout version tag + if err := wt.Checkout(&git.CheckoutOptions{ + Hash: tag.Hash(), + }); err != nil { + s.warn(fmt.Errorf("error checking out tag %s: %w", tag.Name().Short(), err)) + return nil + } + + // Process version packages + packages, err := os.ReadDir(s.i.BasePath) + if err != nil { + s.warn(fmt.Errorf("error reading catalog directory: %w", err)) + return nil + } + for _, p := range packages { + // If the path is not a directory, we skip it + if !p.IsDir() { + continue + } + + // Get package manifest + pkgName := p.Name() + pkgPath := path.Join(s.i.BasePath, pkgName) + manifest, manifestRaw, err := GetManifest(s.i.Repository.Kind, pkgName, pkgPath) + if err != nil { + s.warn(fmt.Errorf("error getting package manifest (path: %s): %w", pkgPath, err)) + continue + } + + // Prepare and store package version + p, err := PreparePackage(&PreparePackageInput{ + R: s.i.Repository, + Tag: tag.Name().Short(), + Manifest: manifest, + ManifestRaw: manifestRaw, + BasePath: s.i.BasePath, + PkgName: pkgName, + PkgPath: pkgPath, + PkgVersion: sv.String(), + }) + if err != nil { + s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, sv.String(), err)) + continue + } + packagesAvailable[pkg.BuildKey(p)] = p + } + + return nil + }) + + return packagesAvailable, nil +} + // warn is a helper that sends the error provided to the errors collector and // logs it as a warning. func (s *TrackerSource) warn(err error) { @@ -206,20 +324,27 @@ func validateManifest(manifest interface{}) error { return errs.ErrorOrNil() } +// PreparePackageInput represents the information required to prepare a package +// of Tekton task and pipelines kinds. +type PreparePackageInput struct { + R *hub.Repository + Tag string + Manifest interface{} + ManifestRaw []byte + BasePath string + PkgName string + PkgPath string + PkgVersion string +} + // PreparePackage prepares a package version using the package manifest and the // files in the path provided. -func PreparePackage( - r *hub.Repository, - manifest interface{}, - manifestRaw []byte, - basePath string, - pkgPath string, -) (*hub.Package, error) { +func PreparePackage(i *PreparePackageInput) (*hub.Package, error) { // Extract some information from package manifest var name, version, description, tektonKind string var annotations map[string]string var tasks []map[string]interface{} - switch m := manifest.(type) { + switch m := i.Manifest.(type) { case *v1beta1.Task: tektonKind = "task" name = m.Name @@ -246,6 +371,9 @@ func PreparePackage( return nil, fmt.Errorf("invalid semver version (%s): %w", version, err) } version = sv.String() + if version != i.PkgVersion { + return nil, fmt.Errorf("version mismatch (%s != %s)", version, i.PkgVersion) + } // Prepare keywords keywords := []string{ @@ -264,16 +392,17 @@ func PreparePackage( DisplayName: annotations[displayNameTKey], Description: description, Keywords: keywords, - Repository: r, + Digest: fmt.Sprintf("%x", sha256.Sum256(i.ManifestRaw)), + Repository: i.R, Data: map[string]interface{}{ PipelinesMinVersionKey: annotations[pipelinesMinVersionTKey], - RawManifestKey: string(manifestRaw), + RawManifestKey: string(i.ManifestRaw), TasksKey: tasks, }, } // Include content and source links - contentURL, sourceURL := prepareContentAndSourceLinks(r, basePath, pkgPath, name) + contentURL, sourceURL := prepareContentAndSourceLinks(i) p.ContentURL = contentURL if sourceURL != "" { p.Links = append(p.Links, &hub.Link{ @@ -293,13 +422,13 @@ func PreparePackage( } // Include readme file - readme, err := os.ReadFile(filepath.Join(pkgPath, "README.md")) + readme, err := os.ReadFile(filepath.Join(i.PkgPath, "README.md")) if err == nil { p.Readme = string(readme) } // Include examples files - examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(pkgPath, examplesPath), nil) + examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(i.PkgPath, examplesPath), nil) if err != nil { if !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("error getting examples files: %w", err) @@ -320,15 +449,10 @@ func PreparePackage( // prepareContentAndSourceLinks prepares the content and source urls for the // git provider identified from the host part in the repository url. -func prepareContentAndSourceLinks( - r *hub.Repository, - basePath string, - pkgPath string, - pkgName string, -) (string, string) { +func prepareContentAndSourceLinks(i *PreparePackageInput) (string, string) { // Parse repository url var repoBaseURL, host, pkgsPath string - matches := repo.GitRepoURLRE.FindStringSubmatch(r.URL) + matches := repo.GitRepoURLRE.FindStringSubmatch(i.R.URL) if len(matches) >= 3 { repoBaseURL = matches[1] host = matches[2] @@ -339,18 +463,21 @@ func prepareContentAndSourceLinks( // Generate content and source url for the corresponding git provider var contentURL, sourceURL string - branch := repo.GetBranch(r) - pkgRelativePath := strings.TrimPrefix(pkgPath, basePath) + branch := i.Tag + if branch == "" { + branch = repo.GetBranch(i.R) + } + pkgRelativePath := strings.TrimPrefix(i.PkgPath, i.BasePath) switch host { case "bitbucket.org": - contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) - sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) + contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) + sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) case "github.com": - contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) - sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) + contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) + sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) case "gitlab.com": - contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) - sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName) + contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) + sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName) } return contentURL, sourceURL diff --git a/internal/tracker/source/tekton/tekton_test.go b/internal/tracker/source/tekton/tekton_test.go index 2ad1d262..2daab002 100644 --- a/internal/tracker/source/tekton/tekton_test.go +++ b/internal/tracker/source/tekton/tekton_test.go @@ -1,6 +1,8 @@ package tekton import ( + "encoding/json" + "fmt" "os" "testing" @@ -19,6 +21,7 @@ func TestTrackerSource(t *testing.T) { i := &hub.TrackerSourceInput{ Repository: &hub.Repository{ Kind: hub.TektonTask, + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), }, BasePath: "testdata/path1", Svc: sw.Svc, @@ -39,6 +42,7 @@ func TestTrackerSource(t *testing.T) { i := &hub.TrackerSourceInput{ Repository: &hub.Repository{ Kind: hub.TektonTask, + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), }, BasePath: "testdata/path2", Svc: sw.Svc, @@ -62,6 +66,7 @@ func TestTrackerSource(t *testing.T) { Repository: &hub.Repository{ Kind: hub.TektonTask, URL: "https://github.com/user/repo/path", + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), }, BasePath: "testdata/path3", Svc: sw.Svc, @@ -79,6 +84,7 @@ func TestTrackerSource(t *testing.T) { Version: "0.1.0", Provider: "Some organization", ContentURL: "https://github.com/user/repo/raw/master/path/task1/0.1/task1.yaml", + Digest: "d5ef3fb05c34644e5ba4fd5a5c3db13be13c11606e663f8583438c2a9d6d243f", Repository: i.Repository, License: "Apache-2.0", Links: []*hub.Link{ @@ -151,6 +157,7 @@ func TestTrackerSource(t *testing.T) { Repository: &hub.Repository{ Kind: hub.TektonPipeline, URL: "https://github.com/user/repo/path", + Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)), }, BasePath: "testdata/path4", Svc: sw.Svc, @@ -167,6 +174,7 @@ func TestTrackerSource(t *testing.T) { Version: "0.1.0", Provider: "Some organization", ContentURL: "https://github.com/user/repo/raw/master/path/pipeline1/0.1/pipeline1.yaml", + Digest: "755a8708c075dbf62529d91495673ef45ad9eedf1cdf0798c97caf8761a69378", Repository: i.Repository, License: "Apache-2.0", Links: []*hub.Link{ diff --git a/web/src/layout/controlPanel/repositories/Modal.tsx b/web/src/layout/controlPanel/repositories/Modal.tsx index 062289fb..175abd95 100644 --- a/web/src/layout/controlPanel/repositories/Modal.tsx +++ b/web/src/layout/controlPanel/repositories/Modal.tsx @@ -10,7 +10,15 @@ import { MdAddCircle } from 'react-icons/md'; import API from '../../../api'; import { AppCtx } from '../../../context/AppCtx'; -import { ContainerTag, ErrorKind, RefInputField, Repository, RepositoryKind, ResourceKind } from '../../../types'; +import { + ContainerTag, + ErrorKind, + RefInputField, + Repository, + RepositoryKind, + ResourceKind, + VersioningOption, +} from '../../../types'; import compoundErrorMessage from '../../../utils/compoundErrorMessage'; import { OCI_PREFIX, RepoKindDef, REPOSITORY_KINDS } from '../../../utils/data'; import getMetaTag from '../../../utils/getMetaTag'; @@ -32,7 +40,9 @@ interface Props { onClose: () => void; onAuthError: () => void; } + const DEFAULT_SELECTED_REPOSITORY_KIND = RepositoryKind.Helm; +const DEFAULT_VERSIONING_OPT = VersioningOption.Directory; const RepositoryModal = (props: Props) => { const { ctx } = useContext(AppCtx); @@ -74,6 +84,11 @@ const RepositoryModal = (props: Props) => { const [containerTags, setContainerTags] = useState(prepareTags()); const [repeatedTagNames, setRepeatedTagNames] = useState(false); + const [versioning, setVersioning] = useState( + props.repository && props.repository.data && props.repository.data.versioning + ? props.repository.data.versioning + : DEFAULT_VERSIONING_OPT + ); const onInputChange = (e: ChangeEvent) => { setIsValidInput(e.target.value === props.repository!.name); @@ -175,6 +190,12 @@ const RepositoryModal = (props: Props) => { tags: readyTags, }; } + + if ([RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind)) { + repository.data = { + versioning: versioning, + }; + } } setIsValidated(true); return { isValid, repository }; @@ -816,6 +837,49 @@ const RepositoryModal = (props: Props) => { )} + {[RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind) && ( + <> + + +
+ {Object.entries(VersioningOption).map((opt: any) => { + return ( +
+ setVersioning(opt[1])} + /> + +
+ ); + })} +
+ +
+ +

+ Select which organization uses your Tekton catalog. For more details please see the{' '} + + Tekton docs + + . +

+ +
+ + )} + {allowPrivateRepositories && ( <> {props.repository && props.repository.private ? ( diff --git a/web/src/types.ts b/web/src/types.ts index 8300eab7..1e3c8d7a 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -38,6 +38,11 @@ export enum Signature { Cosign = 'cosign', } +export enum VersioningOption { + Directory = 'directory', + Git = 'git', +} + export interface Repository { repositoryId?: string; name: string; @@ -61,6 +66,7 @@ export interface Repository { scannerDisabled?: boolean; data?: { tags?: ContainerTag[]; + versioning?: VersioningOption; }; }