hub/internal/tracker/source/tekton/tekton.go

package tekton

import (
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/Masterminds/semver/v3"
	"github.com/artifacthub/hub/internal/hub"
	"github.com/artifacthub/hub/internal/pkg"
	"github.com/artifacthub/hub/internal/repo"
	"github.com/artifacthub/hub/internal/tracker/source"
	"github.com/artifacthub/hub/internal/tracker/source/generic"
	"github.com/ghodss/yaml"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/hashicorp/go-multierror"
	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
)

const (
	// Keys used in labels and annotations in the Tekton's manifest file.

	// displayNameTKey defines the package's display name.
	displayNameTKey = "tekton.dev/displayName"

	// pipelinesMinVersionTKey defines the minimum pipelines version supported.
	pipelinesMinVersionTKey = "tekton.dev/pipelines.minVersion"

	// platformsTKey define the package supported plaatforms.
	platformsTKey = "tekton.dev/platforms"

	// tagsTKey define the package tags.
	tagsTKey = "tekton.dev/tags"

	// versionLabelTKey defines the package version.
	versionLabelTKey = "app.kubernetes.io/version"

	// Keys used in Artifact Hub package's data field.

	// ExamplesKey defines the package examples.
	ExamplesKey = "examples"

	// PipelinesMinVersionKey defines the minimum pipelines version supported.
	PipelinesMinVersionKey = "pipelines.minVersion"

	// PlatformsKey define the package supported plaatforms.
	PlatformsKey = "platforms"

	// RawManifestKey defines the raw manifest.
	RawManifestKey = "manifestRaw"

	// TasksKey defines a list with the pipeline's tasks.
	TasksKey = "tasks"

	// Keys used for Artifact Hub specific annotations.
	changesAnnotation         = "artifacthub.io/changes"
	licenseAnnotation         = "artifacthub.io/license"
	linksAnnotation           = "artifacthub.io/links"
	maintainersAnnotation     = "artifacthub.io/maintainers"
	providerAnnotation        = "artifacthub.io/provider"
	recommendationsAnnotation = "artifacthub.io/recommendations"
	screenshotsAnnotation     = "artifacthub.io/screenshots"

	// examplesPath defines the location of the examples in the package's path.
	examplesPath = "samples"
)

var (
	// errInvalidAnnotation indicates that the annotation provided is not valid.
	errInvalidAnnotation = errors.New("invalid annotation")
)

// TrackerSource is a hub.TrackerSource implementation for Tekton repositories.
type TrackerSource struct {
	i *hub.TrackerSourceInput
}

// NewTrackerSource creates a new TrackerSource instance.
func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource {
	return &TrackerSource{i}
}

// GetPackagesAvailable implements the TrackerSource interface.
func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) {
	// Get repository's Tekton specific data
	if s.i.Repository.Data == nil {
		return nil, errors.New("required repository data field not provided")
	}
	var data *hub.TektonData
	if err := json.Unmarshal(s.i.Repository.Data, &data); err != nil {
		return nil, fmt.Errorf("invalid tekton repository data: %w", err)
	}

	// Process catalog based on the versioning option configured
	switch data.Versioning {
	case hub.TektonDirBasedVersioning:
		return s.processDirBasedCatalog()
	case hub.TektonGitBasedVersioning:
		return s.processGitBasedCatalog()
	default:
		return nil, fmt.Errorf("invalid catalog versioning option: %s", data.Versioning)
	}
}

// processDirBasedCatalog returns the packages available in the catalog using
// the directory based versioning.
func (s *TrackerSource) processDirBasedCatalog() (map[string]*hub.Package, error) {
	packagesAvailable := make(map[string]*hub.Package)

	// Read catalog path to get available packages
	packages, err := os.ReadDir(s.i.BasePath)
	if err != nil {
		return nil, fmt.Errorf("error reading catalog directory: %w", err)
	}
	for _, p := range packages {
		// Return ASAP if context is cancelled
		select {
		case <-s.i.Svc.Ctx.Done():
			return nil, s.i.Svc.Ctx.Err()
		default:
		}

		// If the path is not a directory, we skip it
		if !p.IsDir() {
			continue
		}

		// Read package versions
		pkgName := p.Name()
		pkgBasePath := path.Join(s.i.BasePath, pkgName)
		versions, err := os.ReadDir(pkgBasePath)
		if err != nil {
			s.warn(fmt.Errorf("error reading package %s versions: %w", pkgName, err))
			continue
		}
		for _, v := range versions {
			// If the path is not a directory or a ~valid semver version, we skip it
			if !p.IsDir() {
				continue
			}
			sv, err := semver.NewVersion(v.Name())
			if err != nil {
				continue
			}

			// Get package manifest
			pkgPath := path.Join(pkgBasePath, v.Name())
			manifest, manifestRaw, err := GetManifest(s.i.Repository.Kind, pkgName, pkgPath)
			if err != nil {
				s.warn(fmt.Errorf("error getting package manifest (path: %s): %w", pkgPath, err))
				continue
			}

			// Prepare and store package version
			p, err := PreparePackage(&PreparePackageInput{
				R:           s.i.Repository,
				Tag:         "",
				Manifest:    manifest,
				ManifestRaw: manifestRaw,
				BasePath:    s.i.BasePath,
				PkgName:     pkgName,
				PkgPath:     pkgPath,
				PkgVersion:  sv.String(),
			})
			if err != nil {
				s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, v.Name(), err))
				continue
			}
			packagesAvailable[pkg.BuildKey(p)] = p
		}
	}

	return packagesAvailable, nil
}

// processGitBasedCatalog returns the packages available in the catalog using
// the git based versioning.
func (s *TrackerSource) processGitBasedCatalog() (map[string]*hub.Package, error) {
	// Open git repository and get all tags
	gr, err := git.PlainOpenWithOptions(s.i.BasePath, &git.PlainOpenOptions{
		DetectDotGit: true,
	})
	if err != nil {
		return nil, fmt.Errorf("error opening git repository: %w", err)
	}
	wt, err := gr.Worktree()
	if err != nil {
		return nil, fmt.Errorf("error getting worktree: %w", err)
	}
	tags, err := gr.Tags()
	if err != nil {
		return nil, fmt.Errorf("error reading tags references: %w", err)
	}

	// Read packages available in the catalog for each tag/version
	packagesAvailable := make(map[string]*hub.Package)
	_ = tags.ForEach(func(tag *plumbing.Reference) error {
		// Skip tags that cannot be parsed as ~valid semver
		sv, err := semver.NewVersion(tag.Name().Short())
		if err != nil {
			return nil
		}

		// Checkout version tag
		if err := wt.Checkout(&git.CheckoutOptions{
			Hash: tag.Hash(),
		}); err != nil {
			s.warn(fmt.Errorf("error checking out tag %s: %w", tag.Name().Short(), err))
			return nil
		}

		// Process version packages
		packages, err := os.ReadDir(s.i.BasePath)
		if err != nil {
			s.warn(fmt.Errorf("error reading catalog directory: %w", err))
			return nil
		}
		for _, p := range packages {
			// If the path is not a directory, we skip it
			if !p.IsDir() {
				continue
			}

			// Get package manifest
			pkgName := p.Name()
			pkgPath := path.Join(s.i.BasePath, pkgName)
			manifest, manifestRaw, err := GetManifest(s.i.Repository.Kind, pkgName, pkgPath)
			if err != nil {
				s.warn(fmt.Errorf("error getting package manifest (path: %s): %w", pkgPath, err))
				continue
			}

			// Prepare and store package version
			p, err := PreparePackage(&PreparePackageInput{
				R:           s.i.Repository,
				Tag:         tag.Name().Short(),
				Manifest:    manifest,
				ManifestRaw: manifestRaw,
				BasePath:    s.i.BasePath,
				PkgName:     pkgName,
				PkgPath:     pkgPath,
				PkgVersion:  sv.String(),
			})
			if err != nil {
				s.warn(fmt.Errorf("error preparing package %s version %s: %w", pkgName, sv.String(), err))
				continue
			}
			packagesAvailable[pkg.BuildKey(p)] = p
		}

		return nil
	})

	return packagesAvailable, nil
}

// warn is a helper that sends the error provided to the errors collector and
// logs it as a warning.
func (s *TrackerSource) warn(err error) {
	s.i.Svc.Logger.Warn().Err(err).Send()
	s.i.Svc.Ec.Append(s.i.Repository.RepositoryID, err.Error())
}

// GetManifest reads, parses and validates the package manifest, which can be a
// Tekton task or a pipeline manifest.
func GetManifest(kind hub.RepositoryKind, pkgName, pkgPath string) (interface{}, []byte, error) {
	manifestPath := path.Join(pkgPath, pkgName+".yaml")
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return nil, nil, err
	}
	var manifest interface{}
	switch kind {
	case hub.TektonTask:
		manifest = &v1beta1.Task{}
	case hub.TektonPipeline:
		manifest = &v1beta1.Pipeline{}
	}
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return nil, nil, err
	}
	if err := validateManifest(manifest); err != nil {
		return nil, nil, fmt.Errorf("error validating manifest: %w", err)
	}
	return manifest, manifestData, nil
}

// validateManifest checks if the Tekton manifest provided is valid.
func validateManifest(manifest interface{}) error {
	var errs *multierror.Error

	// Extract some information from package manifest
	var name, version, description string
	switch m := manifest.(type) {
	case *v1beta1.Task:
		name = m.Name
		version = m.Labels[versionLabelTKey]
		description = m.Spec.Description
	case *v1beta1.Pipeline:
		name = m.Name
		version = m.Labels[versionLabelTKey]
		description = m.Spec.Description
	}

	// Validate manifest data
	if name == "" {
		errs = multierror.Append(errs, errors.New("name not provided"))
	}
	if version == "" {
		errs = multierror.Append(errs, errors.New("version not provided"))
	} else if _, err := semver.NewVersion(version); err != nil {
		errs = multierror.Append(errs, fmt.Errorf("invalid version (semver expected): %w", err))
	}
	if description == "" {
		errs = multierror.Append(errs, errors.New("description not provided"))
	}

	return errs.ErrorOrNil()
}

// PreparePackageInput represents the information required to prepare a package
// of Tekton task and pipelines kinds.
type PreparePackageInput struct {
	R           *hub.Repository
	Tag         string
	Manifest    interface{}
	ManifestRaw []byte
	BasePath    string
	PkgName     string
	PkgPath     string
	PkgVersion  string
}

// PreparePackage prepares a package version using the package manifest and the
// files in the path provided.
func PreparePackage(i *PreparePackageInput) (*hub.Package, error) {
	// Extract some information from package manifest
	var name, version, description, tektonKind string
	var annotations map[string]string
	var tasks []map[string]interface{}
	switch m := i.Manifest.(type) {
	case *v1beta1.Task:
		tektonKind = "task"
		name = m.Name
		version = m.Labels[versionLabelTKey]
		description = m.Spec.Description
		annotations = m.Annotations
	case *v1beta1.Pipeline:
		tektonKind = "pipeline"
		name = m.Name
		version = m.Labels[versionLabelTKey]
		description = m.Spec.Description
		annotations = m.Annotations
		for _, task := range m.Spec.Tasks {
			tasks = append(tasks, map[string]interface{}{
				"name":      task.TaskRef.Name,
				"run_after": task.RunAfter,
			})
		}
	}

	// Prepare version
	sv, err := semver.NewVersion(version)
	if err != nil {
		return nil, fmt.Errorf("invalid semver version (%s): %w", version, err)
	}
	version = sv.String()
	if version != i.PkgVersion {
		return nil, fmt.Errorf("version mismatch (%s != %s)", version, i.PkgVersion)
	}

	// Prepare keywords
	keywords := []string{
		"tekton",
		tektonKind,
	}
	tags := strings.Split(annotations[tagsTKey], ",")
	for _, tag := range tags {
		keywords = append(keywords, strings.TrimSpace(tag))
	}

	// Prepare package from manifest information
	p := &hub.Package{
		Name:        name,
		Version:     version,
		DisplayName: annotations[displayNameTKey],
		Description: description,
		Keywords:    keywords,
		Digest:      fmt.Sprintf("%x", sha256.Sum256(i.ManifestRaw)),
		Repository:  i.R,
		Data: map[string]interface{}{
			PipelinesMinVersionKey: annotations[pipelinesMinVersionTKey],
			RawManifestKey:         string(i.ManifestRaw),
			TasksKey:               tasks,
		},
	}

	// Include content and source links
	contentURL, sourceURL := prepareContentAndSourceLinks(i)
	p.ContentURL = contentURL
	if sourceURL != "" {
		p.Links = append(p.Links, &hub.Link{
			Name: "source",
			URL:  sourceURL,
		})
	}

	// Include supported platforms
	if annotations[platformsTKey] != "" {
		tmp := strings.Split(annotations[platformsTKey], ",")
		platforms := make([]string, 0, len(tmp))
		for _, platform := range tmp {
			platforms = append(platforms, strings.TrimSpace(platform))
		}
		p.Data[PlatformsKey] = platforms
	}

	// Include readme file
	readme, err := os.ReadFile(filepath.Join(i.PkgPath, "README.md"))
	if err == nil {
		p.Readme = string(readme)
	}

	// Include examples files
	examples, err := generic.GetFilesWithSuffix(".yaml", path.Join(i.PkgPath, examplesPath), nil)
	if err != nil {
		if !errors.Is(err, os.ErrNotExist) {
			return nil, fmt.Errorf("error getting examples files: %w", err)
		}
	} else {
		if len(examples) > 0 {
			p.Data[ExamplesKey] = examples
		}
	}

	// Enrich package with information from annotations
	if err := enrichPackageFromAnnotations(p, annotations); err != nil {
		return nil, fmt.Errorf("error enriching package %s version %s: %w", name, version, err)
	}

	return p, nil
}

// prepareContentAndSourceLinks prepares the content and source urls for the
// git provider identified from the host part in the repository url.
func prepareContentAndSourceLinks(i *PreparePackageInput) (string, string) {
	// Parse repository url
	var repoBaseURL, host, pkgsPath string
	matches := repo.GitRepoURLRE.FindStringSubmatch(i.R.URL)
	if len(matches) >= 3 {
		repoBaseURL = matches[1]
		host = matches[2]
	}
	if len(matches) == 4 {
		pkgsPath = strings.TrimSuffix(matches[3], "/")
	}

	// Generate content and source url for the corresponding git provider
	var contentURL, sourceURL string
	branch := i.Tag
	if branch == "" {
		branch = repo.GetBranch(i.R)
	}
	pkgRelativePath := strings.TrimPrefix(i.PkgPath, i.BasePath)
	switch host {
	case "bitbucket.org":
		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
		sourceURL = fmt.Sprintf("%s/src/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
	case "github.com":
		contentURL = fmt.Sprintf("%s/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
		sourceURL = fmt.Sprintf("%s/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
	case "gitlab.com":
		contentURL = fmt.Sprintf("%s/-/raw/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
		sourceURL = fmt.Sprintf("%s/-/blob/%s/%s%s/%s.yaml", repoBaseURL, branch, pkgsPath, pkgRelativePath, i.PkgName)
	}

	return contentURL, sourceURL
}

// enrichPackageFromAnnotations adds some extra information to the package from
// the provided annotations.
func enrichPackageFromAnnotations(p *hub.Package, annotations map[string]string) error {
	var errs *multierror.Error

	// Changes
	if v, ok := annotations[changesAnnotation]; ok {
		changes, err := source.ParseChangesAnnotation(v)
		if err != nil {
			errs = multierror.Append(errs, err)
		} else {
			p.Changes = changes
		}
	}

	// License
	p.License = annotations[licenseAnnotation]

	// Links
	if v, ok := annotations[linksAnnotation]; ok {
		var links []*hub.Link
		if err := yaml.Unmarshal([]byte(v), &links); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("%w: invalid links value", errInvalidAnnotation))
		} else {
			p.Links = append(p.Links, links...)
		}
	}

	// Maintainers
	if v, ok := annotations[maintainersAnnotation]; ok {
		var maintainers []*hub.Maintainer
		if err := yaml.Unmarshal([]byte(v), &maintainers); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("%w: invalid maintainers value", errInvalidAnnotation))
		} else {
			var invalidMaintainersFound bool
			for _, maintainer := range maintainers {
				if maintainer.Email == "" {
					invalidMaintainersFound = true
					errs = multierror.Append(errs, fmt.Errorf("%w: maintainer email not provided", errInvalidAnnotation))
				}
			}
			if !invalidMaintainersFound {
				p.Maintainers = maintainers
			}
		}
	}

	// Provider
	p.Provider = annotations[providerAnnotation]

	// Recommendations
	if v, ok := annotations[recommendationsAnnotation]; ok {
		var recommendations []*hub.Recommendation
		if err := yaml.Unmarshal([]byte(v), &recommendations); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("%w: invalid recommendations value", errInvalidAnnotation))
		} else {
			p.Recommendations = recommendations
		}
	}

	// Screenshots
	if v, ok := annotations[screenshotsAnnotation]; ok {
		var screenshots []*hub.Screenshot
		if err := yaml.Unmarshal([]byte(v), &screenshots); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("%w: invalid screenshots value", errInvalidAnnotation))
		} else {
			p.Screenshots = screenshots
		}
	}

	return errs.ErrorOrNil()
}