artifacthub
/
hub
mirror of https://github.com/artifacthub/hub.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add experimental support for Tekton git versioning (#2337) 

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Signed-off-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>
Co-authored-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Co-authored-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>
This commit is contained in:
Sergio Castaño Arteaga 2022-09-20 17:26:48 +02:00 committed by GitHub
parent 86396bc406
commit a138a2b19b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 273 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/ah/lint.go
View File

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
@ -374,6 +375,7 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
repository := &hub.Repository{
Kind: kind,
URL: "https://github.com/user/repo/path",
Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
}
// Read catalog path to get available packages
@ -399,7 +401,8 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
if !p.IsDir() {
continue
}
if _, err := semver.NewVersion(v.Name()); err != nil {
sv, err := semver.NewVersion(v.Name())
if err != nil {
continue
}
@ -416,7 +419,16 @@ func lintTekton(basePath string, kind hub.RepositoryKind) *lintReport {
e.result = multierror.Append(e.result, err)
} else {
// Prepare package version
e.pkg, err = tekton.PreparePackage(repository, manifest, manifestRaw, basePath, pkgPath)
e.pkg, err = tekton.PreparePackage(&tekton.PreparePackageInput{
R: repository,
Tag: "",
Manifest: manifest,
ManifestRaw: manifestRaw,
BasePath: basePath,
PkgName: pkgName,
PkgPath: pkgPath,
PkgVersion: sv.String(),
})
if err != nil {
e.result = multierror.Append(e.result, err)
}

12
internal/hub/repo.go
View File

@ -18,6 +18,12 @@ const (
RepositoryOCIPrefix = "oci://"
)
const (
// Tekton catalog versioning kinds
TektonDirBasedVersioning = "directory"
TektonGitBasedVersioning = "git"
)
// ContainerImageData represents some data specific to repositories of the
// container image kind.
type ContainerImageData struct {
@ -30,6 +36,12 @@ type ContainerImageTag struct {
Mutable bool `json:"mutable"`
}
// TektonData represents some data specific to repositories of the Tekton tasks
// or pipelines kinds.
type TektonData struct {
Versioning string `json:"versioning"` // Options: directory or git
}
// RepositoryKind represents the kind of a given repository.
type RepositoryKind int64

11
internal/repo/manager.go
View File

@ -554,6 +554,17 @@ func (m *Manager) GetRemoteDigest(ctx context.Context, r *hub.Repository) (strin
digest = desc.Digest.String()
case GitRepoURLRE.MatchString(r.URL):
// Do not track repo's digest for Tekton repos using git based versioning
if r.Data != nil {
var data *hub.TektonData
if err := json.Unmarshal(r.Data, &data); err != nil {
return "", fmt.Errorf("invalid tekton repository data: %w", err)
}
if data.Versioning == hub.TektonGitBasedVersioning {
return "", nil
}
}
// Digest is obtained from the last commit in the repository
matches := GitRepoURLRE.FindStringSubmatch(r.URL)
repoBaseURL := matches[1]

187
internal/tracker/source/tekton/tekton.go
View File

@ -1,6 +1,8 @@
package tekton
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
@ -15,6 +17,8 @@ import (
"github.com/artifacthub/hub/internal/tracker/source"
"github.com/artifacthub/hub/internal/tracker/source/generic"
"github.com/ghodss/yaml"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/hashicorp/go-multierror"
"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
)
@ -84,6 +88,29 @@ func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource {
// GetPackagesAvailable implements the TrackerSource interface.
func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) {
// Get repository's Tekton specific data
if s.i.Repository.Data == nil {
return nil, errors.New("required repository data field not provided")
}
var data *hub.TektonData
if err := json.Unmarshal(s.i.Repository.Data, &data); err != nil {
return nil, fmt.Errorf("invalid tekton repository data: %w", err)
}
// Process catalog based on the versioning option configured
switch data.Versioning {
case hub.TektonDirBasedVersioning:
return s.processDirBasedCatalog()
case hub.TektonGitBasedVersioning:
return s.processGitBasedCatalog()
default:
return nil, fmt.Errorf("invalid catalog versioning option: %s", data.Versioning)
}
}
// processDirBasedCatalog returns the packages available in the catalog using
// the directory based versioning.
func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error) {
packagesAvailable := make(map[string]*hub.Package)
// Read catalog path to get available packages
@ -117,7 +144,8 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
if !p.IsDir() {
continue
}
if _, err := semver.NewVersion(v.Name()); err != nil {
sv, err := semver.NewVersion(v.Name())
if err != nil {
continue
}
@ -130,7 +158,16 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
}
// Prepare and store package version
p, err := PreparePackage(s.i.Repository, manifest, manifestRaw, s.i.BasePath, pkgPath)
p, err := PreparePackage(&PreparePackageInput{
R: s.i.Repository,
Tag: "",
Manifest: manifest,
ManifestRaw: manifestRaw,
BasePath: s.i.BasePath,
PkgName: pkgName,
PkgPath: pkgPath,
PkgVersion: sv.String(),
})
if err != nil {
s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, v.Name(), err))
continue
@ -142,6 +179,87 @@ func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error)
return packagesAvailable, nil
}
// processGitBasedCatalog returns the packages available in the catalog using
// the git based versioning.
func (s *TrackerSource) processGitBasedCatalog() (map[string]*hub.Package, error) {
// Open git repository and get all tags
gr, err := git.PlainOpenWithOptions(s.i.BasePath, &git.PlainOpenOptions{
DetectDotGit: true,
})
if err != nil {
return nil, fmt.Errorf("error opening git repository: %w", err)
}
wt, err := gr.Worktree()
if err != nil {
return nil, fmt.Errorf("error getting worktree: %w", err)
}
tags, err := gr.Tags()
if err != nil {
return nil, fmt.Errorf("error reading tags references: %w", err)
}
// Read packages available in the catalog for each tag/version
packagesAvailable := make(map[string]*hub.Package)
_ = tags.ForEach(func(tag *plumbing.Reference) error {
// Skip tags that cannot be parsed as ~valid semver
sv, err := semver.NewVersion(tag.Name().Short())
if err != nil {
return nil
}
// Checkout version tag
if err := wt.Checkout(&git.CheckoutOptions{
Hash: tag.Hash(),
}); err != nil {
s.warn(fmt.Errorf("error checking out tag %s: %w", tag.Name().Short(), err))
return nil
}
// Process version packages
packages, err := os.ReadDir(s.i.BasePath)
if err != nil {
s.warn(fmt.Errorf("error reading catalog directory: %w", err))
return nil
}
for _, p := range packages {
// If the path is not a directory, we skip it
if !p.IsDir() {
continue
}
// Get package manifest
pkgName := p.Name()
pkgPath := path.Join(s.i.BasePath, pkgName)
manifest, manifestRaw, err := GetManifest(s.i.Repository.Kind, pkgName, pkgPath)
if err != nil {
s.warn(fmt.Errorf("error getting package manifest (path: %s): %w", pkgPath, err))
continue
}
// Prepare and store package version
p, err := PreparePackage(&PreparePackageInput{
R: s.i.Repository,
Tag: tag.Name().Short(),
Manifest: manifest,
ManifestRaw: manifestRaw,
BasePath: s.i.BasePath,
PkgName: pkgName,
PkgPath: pkgPath,
PkgVersion: sv.String(),
})
if err != nil {
s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, sv.String(), err))
continue
}
packagesAvailable[pkg.BuildKey(p)] = p
}
return nil
})
return packagesAvailable, nil
}
// warn is a helper that sends the error provided to the errors collector and
// logs it as a warning.
func (s *TrackerSource) warn(err error) {
@ -206,20 +324,27 @@ func validateManifest(manifest interface{}) error {
return errs.ErrorOrNil()
}
// PreparePackageInput represents the information required to prepare a package
// of Tekton task and pipelines kinds.
type PreparePackageInput struct {
R *hub.Repository
Tag string
Manifest interface{}
ManifestRaw []byte
BasePath string
PkgName string
PkgPath string
PkgVersion string
}
// PreparePackage prepares a package version using the package manifest and the
// files in the path provided.
func PreparePackage(
r *hub.Repository,
manifest interface{},
manifestRaw []byte,
basePath string,
pkgPath string,
) (*hub.Package, error) {
func PreparePackage(i *PreparePackageInput) (*hub.Package, error) {
// Extract some information from package manifest
var name, version, description, tektonKind string
var annotations map[string]string
var tasks []map[string]interface{}
switch m := manifest.(type) {
switch m := i.Manifest.(type) {
case *v1beta1.Task:
tektonKind = "task"
name = m.Name
@ -246,6 +371,9 @@ func PreparePackage(
return nil, fmt.Errorf("invalid semver version (%s): %w", version, err)
}
version = sv.String()
if version != i.PkgVersion {
return nil, fmt.Errorf("version mismatch (%s != %s)", version, i.PkgVersion)
}
// Prepare keywords
keywords := []string{
@ -264,16 +392,17 @@ func PreparePackage(
DisplayName: annotations[displayNameTKey],
Description: description,
Keywords: keywords,
Repository: r,
Digest: fmt.Sprintf("%x", sha256.Sum256(i.ManifestRaw)),
Repository: i.R,
Data: map[string]interface{}{
PipelinesMinVersionKey: annotations[pipelinesMinVersionTKey],
RawManifestKey: string(manifestRaw),
RawManifestKey: string(i.ManifestRaw),
TasksKey: tasks,
},
}
// Include content and source links
contentURL, sourceURL := prepareContentAndSourceLinks(r, basePath, pkgPath, name)
contentURL, sourceURL := prepareContentAndSourceLinks(i)
p.ContentURL = contentURL
if sourceURL != "" {
p.Links = append(p.Links, &hub.Link{
@ -293,13 +422,13 @@ func PreparePackage(
}
// Include readme file
readme, err := os.ReadFile(filepath.Join(pkgPath, "README.md"))
readme, err := os.ReadFile(filepath.Join(i.PkgPath, "README.md"))
if err == nil {
p.Readme = string(readme)
}
// Include examples files
examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(pkgPath, examplesPath), nil)
examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(i.PkgPath, examplesPath), nil)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("error getting examples files: %w", err)
@ -320,15 +449,10 @@ func PreparePackage(
// prepareContentAndSourceLinks prepares the content and source urls for the
// git provider identified from the host part in the repository url.
func prepareContentAndSourceLinks(
r *hub.Repository,
basePath string,
pkgPath string,
pkgName string,
) (string, string) {
func prepareContentAndSourceLinks(i *PreparePackageInput) (string, string) {
// Parse repository url
var repoBaseURL, host, pkgsPath string
matches := repo.GitRepoURLRE.FindStringSubmatch(r.URL)
matches := repo.GitRepoURLRE.FindStringSubmatch(i.R.URL)
if len(matches) >= 3 {
repoBaseURL = matches[1]
host = matches[2]
@ -339,18 +463,21 @@ func prepareContentAndSourceLinks(
// Generate content and source url for the corresponding git provider
var contentURL, sourceURL string
branch := repo.GetBranch(r)
pkgRelativePath := strings.TrimPrefix(pkgPath, basePath)
branch := i.Tag
if branch == "" {
branch = repo.GetBranch(i.R)
}
pkgRelativePath := strings.TrimPrefix(i.PkgPath, i.BasePath)
switch host {
case "bitbucket.org":
contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
case "github.com":
contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
case "gitlab.com":
contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, pkgName)
contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
}
return contentURL, sourceURL

8
internal/tracker/source/tekton/tekton_test.go
View File

@ -1,6 +1,8 @@
package tekton
import (
"encoding/json"
"fmt"
"os"
"testing"
@ -19,6 +21,7 @@ func TestTrackerSource(t *testing.T) {
i := &hub.TrackerSourceInput{
Repository: &hub.Repository{
Kind: hub.TektonTask,
Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
},
BasePath: "testdata/path1",
Svc: sw.Svc,
@ -39,6 +42,7 @@ func TestTrackerSource(t *testing.T) {
i := &hub.TrackerSourceInput{
Repository: &hub.Repository{
Kind: hub.TektonTask,
Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
},
BasePath: "testdata/path2",
Svc: sw.Svc,
@ -62,6 +66,7 @@ func TestTrackerSource(t *testing.T) {
Repository: &hub.Repository{
Kind: hub.TektonTask,
URL: "https://github.com/user/repo/path",
Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
},
BasePath: "testdata/path3",
Svc: sw.Svc,
@ -79,6 +84,7 @@ func TestTrackerSource(t *testing.T) {
Version: "0.1.0",
Provider: "Some organization",
ContentURL: "https://github.com/user/repo/raw/master/path/task1/0.1/task1.yaml",
Digest: "d5ef3fb05c34644e5ba4fd5a5c3db13be13c11606e663f8583438c2a9d6d243f",
Repository: i.Repository,
License: "Apache-2.0",
Links: []*hub.Link{
@ -151,6 +157,7 @@ func TestTrackerSource(t *testing.T) {
Repository: &hub.Repository{
Kind: hub.TektonPipeline,
URL: "https://github.com/user/repo/path",
Data: json.RawMessage(fmt.Sprintf(`{"versioning": "%s"}`, hub.TektonDirBasedVersioning)),
},
BasePath: "testdata/path4",
Svc: sw.Svc,
@ -167,6 +174,7 @@ func TestTrackerSource(t *testing.T) {
Version: "0.1.0",
Provider: "Some organization",
ContentURL: "https://github.com/user/repo/raw/master/path/pipeline1/0.1/pipeline1.yaml",
Digest: "755a8708c075dbf62529d91495673ef45ad9eedf1cdf0798c97caf8761a69378",
Repository: i.Repository,
License: "Apache-2.0",
Links: []*hub.Link{

66
web/src/layout/controlPanel/repositories/Modal.tsx
View File

@ -10,7 +10,15 @@ import { MdAddCircle } from 'react-icons/md';
import API from '../../../api';
import { AppCtx } from '../../../context/AppCtx';
import { ContainerTag, ErrorKind, RefInputField, Repository, RepositoryKind, ResourceKind } from '../../../types';
import {
ContainerTag,
ErrorKind,
RefInputField,
Repository,
RepositoryKind,
ResourceKind,
VersioningOption,
} from '../../../types';
import compoundErrorMessage from '../../../utils/compoundErrorMessage';
import { OCI_PREFIX, RepoKindDef, REPOSITORY_KINDS } from '../../../utils/data';
import getMetaTag from '../../../utils/getMetaTag';
@ -32,7 +40,9 @@ interface Props {
onClose: () => void;
onAuthError: () => void;
}
const DEFAULT_SELECTED_REPOSITORY_KIND = RepositoryKind.Helm;
const DEFAULT_VERSIONING_OPT = VersioningOption.Directory;
const RepositoryModal = (props: Props) => {
const { ctx } = useContext(AppCtx);
@ -74,6 +84,11 @@ const RepositoryModal = (props: Props) => {
const [containerTags, setContainerTags] = useState<ContainerTag[]>(prepareTags());
const [repeatedTagNames, setRepeatedTagNames] = useState<boolean>(false);
const [versioning, setVersioning] = useState<VersioningOption>(
props.repository && props.repository.data && props.repository.data.versioning
? props.repository.data.versioning
: DEFAULT_VERSIONING_OPT
);
const onInputChange = (e: ChangeEvent<HTMLInputElement>) => {
setIsValidInput(e.target.value === props.repository!.name);
@ -175,6 +190,12 @@ const RepositoryModal = (props: Props) => {
tags: readyTags,
};
}
if ([RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind)) {
repository.data = {
versioning: versioning,
};
}
}
setIsValidated(true);
return { isValid, repository };
@ -816,6 +837,49 @@ const RepositoryModal = (props: Props) => {
</div>
)}
{[RepositoryKind.TektonTask, RepositoryKind.TektonPipeline].includes(selectedKind) && (
<>
<label className={`form-label fw-bold ${styles.label}`}>Versioning</label>
<div className="d-flex flex-row mb-2">
{Object.entries(VersioningOption).map((opt: any) => {
return (
<div className="form-check me-4 mb-2" key={`versioning_${opt[1]}`}>
<input
className="form-check-input"
type="radio"
id={opt[1]}
name="payload"
value={opt[1]}
checked={versioning === opt[1]}
onChange={() => setVersioning(opt[1])}
/>
<label className={`form-check-label ${styles.label}`} htmlFor={opt[1]}>
{opt[0]}
</label>
</div>
);
})}
</div>
<div className="mb-4">
<small className="text-muted text-break">
<p className="mb-0">
Select which organization uses your Tekton catalog. For more details please see the{' '}
<ExternalLink
href="https://github.com/tektoncd/community/blob/main/teps/0115-tekton-catalog-git-based-versioning.md#tep-0115-tekton-catalog-git-based-versioning"
className="text-primary fw-bold"
label="Open Tekton docs"
>
Tekton docs
</ExternalLink>
.
</p>
</small>
</div>
</>
)}
{allowPrivateRepositories && (
<>
{props.repository && props.repository.private ? (

6
web/src/types.ts
View File

@ -38,6 +38,11 @@ export enum Signature {
Cosign = 'cosign',
}
export enum VersioningOption {
Directory = 'directory',
Git = 'git',
}
export interface Repository {
repositoryId?: string;
name: string;
@ -61,6 +66,7 @@ export interface Repository {
scannerDisabled?: boolean;
data?: {
tags?: ContainerTag[];
versioning?: VersioningOption;
};
}