Add experimental support for Tekton git versioning (#2337)

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Signed-off-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>
Co-authored-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Co-authored-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>

cmd/ah/lint.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -374,6 +375,7 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
 	repository := &hub.Repository{
 		Kind: kind,
 		URL:  "https://github.com/user/repo/path",
+		Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
 	}
 
 	// Read catalog path to get available packages
@@ -399,7 +401,8 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
 			if !p.IsDir() {
 				continue
 			}
-			if _, err := semver.NewVersion(v.Name()); err != nil {
+			sv, err := semver.NewVersion(v.Name())
+			if err != nil {
 				continue
 			}
 
@@ -416,7 +419,16 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
 				e.result = multierror.Append(e.result, err)
 			} else {
 				// Prepare package version
-				e.pkg, err = tekton.PreparePackage(repository, manifest, manifestRaw, basePath, pkgPath)
+				e.pkg, err = tekton.PreparePackage(&tekton.PreparePackageInput{
+					R:           repository,
+					Tag:         "",
+					Manifest:    manifest,
+					ManifestRaw: manifestRaw,
+					BasePath:    basePath,
+					PkgName:     pkgName,
+					PkgPath:     pkgPath,
+					PkgVersion:  sv.String(),
+				})
 				if err != nil {
 					e.result = multierror.Append(e.result, err)
 				}

internal/hub/repo.go

@@ -18,6 +18,12 @@ const (
 	RepositoryOCIPrefix = "oci://"
 )
 
+const (
+	// Tekton catalog versioning kinds
+	TektonDirBasedVersioning = "directory"
+	TektonGitBasedVersioning = "git"
+)
+
 // ContainerImageData represents some data specific to repositories of the
 // container image kind.
 type ContainerImageData struct {
@@ -30,6 +36,12 @@ type ContainerImageTag struct {
 	Mutable bool   `json:"mutable"`
 }
 
+// TektonData represents some data specific to repositories of the Tekton tasks
+// or pipelines kinds.
+type TektonData struct {
+	Versioning string `json:"versioning"` // Options: directory or git
+}
+
 // RepositoryKind represents the kind of a given repository.
 type RepositoryKind int64
 

internal/repo/manager.go

@@ -554,6 +554,17 @@ func (m *Manager) GetRemoteDigest(ctx context.Context, r *hub.Repository) (strin
 		digest = desc.Digest.String()
 
 	case GitRepoURLRE.MatchString(r.URL):
+		// Do not track repo's digest for Tekton repos using git based versioning
+		if r.Data != nil {
+			var data *hub.TektonData
+			if err := json.Unmarshal(r.Data, &data); err != nil {
+				return "", fmt.Errorf("invalid tekton repository data: %w", err)
+			}
+			if data.Versioning == hub.TektonGitBasedVersioning {
+				return "", nil
+			}
+		}
+
 		// Digest is obtained from the last commit in the repository
 		matches := GitRepoURLRE.FindStringSubmatch(r.URL)
 		repoBaseURL := matches[1]

internal/tracker/source/tekton/tekton.go

@@ -1,6 +1,8 @@
 package tekton
 
 import (
+	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -15,6 +17,8 @@ import (
 	"github.com/artifacthub/hub/internal/tracker/source"
 	"github.com/artifacthub/hub/internal/tracker/source/generic"
 	"github.com/ghodss/yaml"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/hashicorp/go-multierror"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
 )
@@ -84,6 +88,29 @@ func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource {
 
 // GetPackagesAvailable implements the TrackerSource interface.
 func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) {
+	// Get repository's Tekton specific data
+	if s.i.Repository.Data == nil {
+		return nil, errors.New("required repository data field not provided")
+	}
+	var data *hub.TektonData
+	if err := json.Unmarshal(s.i.Repository.Data, &data); err != nil {
+		return nil, fmt.Errorf("invalid tekton repository data: %w", err)
+	}
+
+	// Process catalog based on the versioning option configured
+	switch data.Versioning {
+	case hub.TektonDirBasedVersioning:
+		return s.processDirBasedCatalog()
+	case hub.TektonGitBasedVersioning:
+		return s.processGitBasedCatalog()
+	default:
+		return nil, fmt.Errorf("invalid catalog versioning option: %s", data.Versioning)
+	}
+}
+
+// processDirBasedCatalog returns the packages available in the catalog using
+// the directory based versioning.
+func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error) {
 	packagesAvailable := make(map[string]*hub.Package)
 
 	// Read catalog path to get available packages
@@ -117,7 +144,8 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
 			if !p.IsDir() {
 				continue
 			}
-			if _, err := semver.NewVersion(v.Name()); err != nil {
+			sv, err := semver.NewVersion(v.Name())
+			if err != nil {
 				continue
 			}
 
@@ -130,7 +158,16 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
 			}
 
 			// Prepare and store package version
-			p, err := PreparePackage(s.i.Repository, manifest, manifestRaw, s.i.BasePath, pkgPath)
+			p, err := PreparePackage(&PreparePackageInput{
+				R:           s.i.Repository,
+				Tag:         "",
+				Manifest:    manifest,
+				ManifestRaw: manifestRaw,
+				BasePath:    s.i.BasePath,
+				PkgName:     pkgName,
+				PkgPath:     pkgPath,
+				PkgVersion:  sv.String(),
+			})
 			if err != nil {
 				s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, v.Name(), err))
 				continue
@@ -142,6 +179,87 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
 	return packagesAvailable, nil
 }
 
+// processGitBasedCatalog returns the packages available in the catalog using
+// the git based versioning.
+func (s *TrackerSource) processGitBasedCatalog() (map[string]*hub.Package, error) {
+	// Open git repository and get all tags
+	gr, err := git.PlainOpenWithOptions(s.i.BasePath, &git.PlainOpenOptions{
+		DetectDotGit: true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error opening git repository: %w", err)
+	}
+	wt, err := gr.Worktree()
+	if err != nil {
+		return nil, fmt.Errorf("error getting worktree: %w", err)
+	}
+	tags, err := gr.Tags()
+	if err != nil {
+		return nil, fmt.Errorf("error reading tags references: %w", err)
+	}
+
+	// Read packages available in the catalog for each tag/version
+	packagesAvailable := make(map[string]*hub.Package)
+	_ = tags.ForEach(func(tag *plumbing.Reference) error {
+		// Skip tags that cannot be parsed as ~valid semver
+		sv, err := semver.NewVersion(tag.Name().Short())
+		if err != nil {
+			return nil
+		}
+
+		// Checkout version tag
+		if err := wt.Checkout(&git.CheckoutOptions{
+			Hash: tag.Hash(),
+		}); err != nil {
+			s.warn(fmt.Errorf("error checking out tag %s: %w", tag.Name().Short(), err))
+			return nil
+		}
+
+		// Process version packages
+		packages, err := os.ReadDir(s.i.BasePath)
+		if err != nil {
+			s.warn(fmt.Errorf("error reading catalog directory: %w", err))
+			return nil
+		}
+		for _, p := range packages {
+			// If the path is not a directory, we skip it
+			if !p.IsDir() {
+				continue
+			}
+
+			// Get package manifest
+			pkgName := p.Name()
+			pkgPath := path.Join(s.i.BasePath, pkgName)
+			manifest, manifestRaw, err := GetManifest(s.i.Repository.Kind, pkgName, pkgPath)
+			if err != nil {
+				s.warn(fmt.Errorf("error getting package manifest (path: %s): %w", pkgPath, err))
+				continue
+			}
+
+			// Prepare and store package version
+			p, err := PreparePackage(&PreparePackageInput{
+				R:           s.i.Repository,
+				Tag:         tag.Name().Short(),
+				Manifest:    manifest,
+				ManifestRaw: manifestRaw,
+				BasePath:    s.i.BasePath,
+				PkgName:     pkgName,
+				PkgPath:     pkgPath,
+				PkgVersion:  sv.String(),
+			})
+			if err != nil {
+				s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, sv.String(), err))
+				continue
+			}
+			packagesAvailable[pkg.BuildKey(p)] = p
+		}
+
+		return nil
+	})
+
+	return packagesAvailable, nil
+}
+
 // warn is a helper that sends the error provided to the errors collector and
 // logs it as a warning.
 func (s *TrackerSource) warn(err error) {
@@ -206,20 +324,27 @@ func validateManifest(manifest interface{}) error {
 	return errs.ErrorOrNil()
 }
 
+// PreparePackageInput represents the information required to prepare a package
+// of Tekton task and pipelines kinds.
+type PreparePackageInput struct {
+	R           *hub.Repository
+	Tag         string
+	Manifest    interface{}
+	ManifestRaw []byte
+	BasePath    string
+	PkgName     string
+	PkgPath     string
+	PkgVersion  string
+}
+
 // PreparePackage prepares a package version using the package manifest and the
 // files in the path provided.
-func PreparePackage(
+func PreparePackage(i *PreparePackageInput) (*hub.Package, error) {
-	r *hub.Repository,
-	manifest interface{},
-	manifestRaw []byte,
-	basePath string,
-	pkgPath string,
-) (*hub.Package, error) {
 	// Extract some information from package manifest
 	var name, version, description, tektonKind string
 	var annotations map[string]string
 	var tasks []map[string]interface{}
-	switch m := manifest.(type) {
+	switch m := i.Manifest.(type) {
 	case *v1beta1.Task:
 		tektonKind = "task"
 		name = m.Name
@@ -246,6 +371,9 @@ func PreparePackage(
 		return nil, fmt.Errorf("invalid semver version (%s): %w", version, err)
 	}
 	version = sv.String()
+	if version != i.PkgVersion {
+		return nil, fmt.Errorf("version mismatch (%s != %s)", version, i.PkgVersion)
+	}
 
 	// Prepare keywords
 	keywords := []string{
@@ -264,16 +392,17 @@ func PreparePackage(
 		DisplayName: annotations[displayNameTKey],
 		Description: description,
 		Keywords:    keywords,
-		Repository:  r,
+		Digest:      fmt.Sprintf("%x", sha256.Sum256(i.ManifestRaw)),
+		Repository:  i.R,
 		Data: map[string]interface{}{
 			PipelinesMinVersionKey: annotations[pipelinesMinVersionTKey],
-			RawManifestKey:         string(manifestRaw),
+			RawManifestKey:         string(i.ManifestRaw),
 			TasksKey:               tasks,
 		},
 	}
 
 	// Include content and source links
-	contentURL, sourceURL := prepareContentAndSourceLinks(r, basePath, pkgPath, name)
+	contentURL, sourceURL := prepareContentAndSourceLinks(i)
 	p.ContentURL = contentURL
 	if sourceURL != "" {
 		p.Links = append(p.Links, &hub.Link{
@@ -293,13 +422,13 @@ func PreparePackage(
 	}
 
 	// Include readme file
-	readme, err := os.ReadFile(filepath.Join(pkgPath, "README.md"))
+	readme, err := os.ReadFile(filepath.Join(i.PkgPath, "README.md"))
 	if err == nil {
 		p.Readme = string(readme)
 	}
 
 	// Include examples files
-	examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(pkgPath, examplesPath), nil)
+	examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(i.PkgPath, examplesPath), nil)
 	if err != nil {
 		if !errors.Is(err, os.ErrNotExist) {
 			return nil, fmt.Errorf("error getting examples files: %w", err)
@@ -320,15 +449,10 @@ func PreparePackage(
 
 // prepareContentAndSourceLinks prepares the content and source urls for the
 // git provider identified from the host part in the repository url.
-func prepareContentAndSourceLinks(
+func prepareContentAndSourceLinks(i *PreparePackageInput) (string, string) {
-	r *hub.Repository,
-	basePath string,
-	pkgPath string,
-	pkgName string,
-) (string, string) {
 	// Parse repository url
 	var repoBaseURL, host, pkgsPath string
-	matches := repo.GitRepoURLRE.FindStringSubmatch(r.URL)
+	matches := repo.GitRepoURLRE.FindStringSubmatch(i.R.URL)
 	if len(matches) >= 3 {
 		repoBaseURL = matches[1]
 		host = matches[2]
@@ -339,18 +463,21 @@ func prepareContentAndSourceLinks(
 
 	// Generate content and source url for the corresponding git provider
 	var contentURL, sourceURL string
-	branch := repo.GetBranch(r)
+	branch := i.Tag
-	pkgRelativePath := strings.TrimPrefix(pkgPath, basePath)
+	if branch == "" {
+		branch = repo.GetBranch(i.R)
+	}
+	pkgRelativePath := strings.TrimPrefix(i.PkgPath, i.BasePath)
 	switch host {
 	case "bitbucket.org":
-		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
-		sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
 	case "github.com":
-		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
-		sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
 	case "gitlab.com":
-		contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
-		sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
+		sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
 	}
 
 	return contentURL, sourceURL

internal/tracker/source/tekton/tekton_test.go

@@ -1,6 +1,8 @@
 package tekton
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 	"testing"
 
@@ -19,6 +21,7 @@ func TestTrackerSource(t *testing.T) {
 		i := &hub.TrackerSourceInput{
 			Repository: &hub.Repository{
 				Kind: hub.TektonTask,
+				Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
 			},
 			BasePath: "testdata/path1",
 			Svc:      sw.Svc,
@@ -39,6 +42,7 @@ func TestTrackerSource(t *testing.T) {
 		i := &hub.TrackerSourceInput{
 			Repository: &hub.Repository{
 				Kind: hub.TektonTask,
+				Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
 			},
 			BasePath: "testdata/path2",
 			Svc:      sw.Svc,
@@ -62,6 +66,7 @@ func TestTrackerSource(t *testing.T) {
 			Repository: &hub.Repository{
 				Kind: hub.TektonTask,
 				URL:  "https://github.com/user/repo/path",
+				Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
 			},
 			BasePath: "testdata/path3",
 			Svc:      sw.Svc,
@@ -79,6 +84,7 @@ func TestTrackerSource(t *testing.T) {
 			Version:     "0.1.0",
 			Provider:    "Some organization",
 			ContentURL:  "https://github.com/user/repo/raw/master/path/task1/0.1/task1.yaml",
+			Digest:      "d5ef3fb05c34644e5ba4fd5a5c3db13be13c11606e663f8583438c2a9d6d243f",
 			Repository:  i.Repository,
 			License:     "Apache-2.0",
 			Links: []*hub.Link{
@@ -151,6 +157,7 @@ func TestTrackerSource(t *testing.T) {
 			Repository: &hub.Repository{
 				Kind: hub.TektonPipeline,
 				URL:  "https://github.com/user/repo/path",
+				Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
 			},
 			BasePath: "testdata/path4",
 			Svc:      sw.Svc,
@@ -167,6 +174,7 @@ func TestTrackerSource(t *testing.T) {
 			Version:     "0.1.0",
 			Provider:    "Some organization",
 			ContentURL:  "https://github.com/user/repo/raw/master/path/pipeline1/0.1/pipeline1.yaml",
+			Digest:      "755a8708c075dbf62529d91495673ef45ad9eedf1cdf0798c97caf8761a69378",
 			Repository:  i.Repository,
 			License:     "Apache-2.0",
 			Links: []*hub.Link{

web/src/layout/controlPanel/repositories/Modal.tsx

@@ -10,7 +10,15 @@ import { MdAddCircle } from 'react-icons/md';
 
 import API from '../../../api';
 import { AppCtx } from '../../../context/AppCtx';
-import { ContainerTag, ErrorKind, RefInputField, Repository, RepositoryKind, ResourceKind } from '../../../types';
+import {
+  ContainerTag,
+  ErrorKind,
+  RefInputField,
+  Repository,
+  RepositoryKind,
+  ResourceKind,
+  VersioningOption,
+} from '../../../types';
 import compoundErrorMessage from '../../../utils/compoundErrorMessage';
 import { OCI_PREFIX, RepoKindDef, REPOSITORY_KINDS } from '../../../utils/data';
 import getMetaTag from '../../../utils/getMetaTag';
@@ -32,7 +40,9 @@ interface Props {
   onClose: () => void;
   onAuthError: () => void;
 }
+
 const DEFAULT_SELECTED_REPOSITORY_KIND = RepositoryKind.Helm;
+const DEFAULT_VERSIONING_OPT = VersioningOption.Directory;
 
 const RepositoryModal = (props: Props) => {
   const { ctx } = useContext(AppCtx);
@@ -74,6 +84,11 @@ const RepositoryModal = (props: Props) => {
 
   const [containerTags, setContainerTags] = useState<ContainerTag[]>(prepareTags());
   const [repeatedTagNames, setRepeatedTagNames] = useState<boolean>(false);
+  const [versioning, setVersioning] = useState<VersioningOption>(
+    props.repository && props.repository.data && props.repository.data.versioning
+      ? props.repository.data.versioning
+      : DEFAULT_VERSIONING_OPT
+  );
 
   const onInputChange = (e: ChangeEvent<HTMLInputElement>) => {
     setIsValidInput(e.target.value === props.repository!.name);
@@ -175,6 +190,12 @@ const RepositoryModal = (props: Props) => {
             tags: readyTags,
           };
         }
+
+        if ([RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind)) {
+          repository.data = {
+            versioning: versioning,
+          };
+        }
       }
       setIsValidated(true);
       return { isValid, repository };
@@ -816,6 +837,49 @@ const RepositoryModal = (props: Props) => {
               </div>
             )}
 
+            {[RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind) && (
+              <>
+                <label className={`form-label fw-bold ${styles.label}`}>Versioning</label>
+
+                <div className="d-flex flex-row mb-2">
+                  {Object.entries(VersioningOption).map((opt: any) => {
+                    return (
+                      <div className="form-check me-4 mb-2" key={`versioning_${opt[1]}`}>
+                        <input
+                          className="form-check-input"
+                          type="radio"
+                          id={opt[1]}
+                          name="payload"
+                          value={opt[1]}
+                          checked={versioning === opt[1]}
+                          onChange={() => setVersioning(opt[1])}
+                        />
+                        <label className={`form-check-label ${styles.label}`} htmlFor={opt[1]}>
+                          {opt[0]}
+                        </label>
+                      </div>
+                    );
+                  })}
+                </div>
+
+                <div className="mb-4">
+                  <small className="text-muted text-break">
+                    <p className="mb-0">
+                      Select which organization uses your Tekton catalog. For more details please see the{' '}
+                      <ExternalLink
+                        href="https://github.com/tektoncd/community/blob/main/teps/0115-tekton-catalog-git-based-versioning.md#tep-0115-tekton-catalog-git-based-versioning"
+                        className="text-primary fw-bold"
+                        label="Open Tekton docs"
+                      >
+                        Tekton docs
+                      </ExternalLink>
+                      .
+                    </p>
+                  </small>
+                </div>
+              </>
+            )}
+
             {allowPrivateRepositories && (
               <>
                 {props.repository && props.repository.private ? (

web/src/types.ts

@@ -38,6 +38,11 @@ export enum Signature {
   Cosign = 'cosign',
 }
 
+export enum VersioningOption {
+  Directory = 'directory',
+  Git = 'git',
+}
+
 export interface Repository {
   repositoryId?: string;
   name: string;
@@ -61,6 +66,7 @@ export interface Repository {
   scannerDisabled?: boolean;
   data?: {
     tags?: ContainerTag[];
+    versioning?: VersioningOption;
   };
 }