func/pkg/functions/repository.go

package functions

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/storage/memory"
	"gopkg.in/yaml.v2"

	"knative.dev/func/pkg/filesystem"
)

const (
	manifestFile = "manifest.yaml"

	// DefaultReadinessEndpoint for final deployed function instances
	DefaultReadinessEndpoint = "/health/readiness"

	// DefaultLivenessEndpoint for final deployed function instances
	DefaultLivenessEndpoint = "/health/liveness"

	// DefaultInvocationFormat is a named invocation hint for the convenience
	// helper .Invoke.  It is usually set at the template level.  The default
	// ('http') is a plain HTTP POST.
	DefaultInvocationFormat = "http"

	// Defaults for Builder and Builders not expressly defined as a purposeful
	// delegation of choice.
)

// Repository can be local or remote and contains runtimes
type Repository struct {
	// Runtimes containing Templates loaded from the repo
	Runtimes []Runtime

	repoConfig // values defineable via a manfest.yaml at root level.

	fs  filesystem.Filesystem
	uri string // populated on initial add
}

// Runtime contains templates
type Runtime struct {
	// Name of the runtime
	Name string

	// Templates defined for the runtime
	Templates []Template

	config runtimeConfig // manifest.yaml at root plus runtime level.
}

// see template.go for template definition
// It's config is the root manifest.yaml + runtime manifest.yaml plus
// the template-level.

// repoConfig is the manifest.yaml at the repository level, and
// contains all settings from the runtimeConfig plus a few for the repo.
type repoConfig struct {
	// repository manifest.yaml can define some default values for func.yaml
	runtimeConfig `yaml:",inline"`

	// Name is either directory name on FS or last part of git URL or
	// arbitrary value defined by the Template author or as indicated by the
	// repository author via manifest.yaml.
	Name string `yaml:"name,omitempty"`

	// Version of the repository.
	Version string `yaml:"version,omitempty"`

	// TemplatesPath defines an optional path within the repository at which
	// templates are stored.  By default this is the repository root.
	TemplatesPath string `yaml:"templates,omitempty"`
}

// runtimeConfig is the manifest.yaml at the runtime level, and contains
// all settings from the funcConfig.
type runtimeConfig struct {
	templateConfig `yaml:",inline"`
	// runtime currently contains no level-specific attributes.
}

// templateConfig is the manifest.yaml file in a template directory or the
// runtime parent directory, and is used during template.Write() to set
// values on the newly written Function.
type templateConfig struct {
	// BuildConfig defines builders and buildpacks.  the denormalized view of
	// members which can be defined per repo or per runtime first.
	BuildConfig `yaml:",inline"`

	// HealthEndpoints.  The denormalized view of members which can be defined
	// first per repo or per runtime.
	HealthEndpoints `yaml:"healthEndpoints,omitempty"`

	// BuildEnvs defines environment variables related to the builders,
	// this can be used to parameterize the builders
	BuildEnvs []Env `yaml:"buildEnvs,omitempty"`

	// RunEnvs defines environment variables used in runtime.
	RunEnvs []Env `yaml:"runEnvs,omitempty"`

	// Invoke defines invocation hints for a functions which is created
	// from this template prior to being materially modified.
	Invoke string `yaml:"invoke,omitempty"`
}

// NewRepository creates a repository instance from any of: a path on disk, a
// remote or local URI, or from the embedded default repo if uri not provided.
// Name (optional), if provided takes precedence over name derived from repo at
// the given URI.
//
// uri (optional), the path either locally or remote from which to load
// the repository files.  If not provided, the internal default is assumed.
func NewRepository(name, uri string) (repo Repository, err error) {
	repo = Repository{uri: uri}

	repo.Name, err = repositoryName(name, uri)
	if err != nil {
		return
	}

	repo.fs, err = filesystemFromURI(uri)
	if err != nil {
		return Repository{}, fmt.Errorf("failed to get repository from URI (%q): %w", uri, err)
	}

	repo.repoConfig, err = loadRepoConfig(repo.fs, repo.repoConfig)
	if err != nil {
		return
	}

	repo.Runtimes, err = runtimes(repo.fs, repo.repoConfig)
	return
}

// FS returns the underlying filesystem of this repository.
func (r Repository) FS() filesystem.Filesystem {
	return r.fs
}

// filesystemFromURI returns a filesystem from the data located at the
// given URI.  If URI is not provided, indicates the embedded repo should
// be loaded.  URI can be a remote git repository (http:// https:// etc.),
// or a local file path (file://) which can be a git repo or a plain directory.
func filesystemFromURI(uri string) (f filesystem.Filesystem, err error) {
	// If not provided, indicates embedded.
	if uri == "" {
		return EmbeddedTemplatesFS, nil
	}

	if isNonBareGitRepo(uri) {
		return filesystemFromPath(uri)
	}

	// Attempt to get a filesystem from the uri as a remote repo.
	f, err = FilesystemFromRepo(uri)
	if f != nil || err != nil {
		return // found a filesystem and/or an error
	}

	// Attempt to get a filesystem from the uri as a file path.
	return filesystemFromPath(uri)
}

func isNonBareGitRepo(uri string) bool {
	parsed, err := url.Parse(uri)
	if err != nil {
		return false
	}
	if parsed.Scheme != "file" {
		return false
	}
	p := filepath.Join(filepath.FromSlash(uri[7:]), ".git")
	fi, err := os.Stat(p)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

// FilesystemFromRepo attempts to fetch a filesystem from a git repository
// indicated by the given URI.  Returns nil if there is not a repo at the URI.
func FilesystemFromRepo(uri string) (filesystem.Filesystem, error) {
	clone, err := git.Clone(
		memory.NewStorage(),
		memfs.New(),
		getGitCloneOptions(uri),
	)
	if err != nil {
		if isRepoNotFoundError(err) {
			return nil, nil
		}
		if isBranchNotFoundError(err) {
			return nil, fmt.Errorf("failed to clone repository: branch not found for uri %s", uri)
		}
		return nil, fmt.Errorf("failed to clone repository: %w", err)
	}
	wt, err := clone.Worktree()
	if err != nil {
		return nil, err
	}
	return filesystem.NewBillyFilesystem(wt.Filesystem), nil
}

// isRepoNotFoundError returns true if the error is a
// "repository not found" error.
func isRepoNotFoundError(err error) bool {
	// This would be better if the error being tested for was typed, but it is
	// currently a simple string value comparison.
	return (err != nil && err.Error() == "repository not found")
}

func isBranchNotFoundError(err error) bool {
	// This would be better if the error being tested for was typed, but it is
	// currently a simple string value comparison.
	return (err != nil && err.Error() == "reference not found")
}

// filesystemFromPath attempts to return a filesystem from a URI as a file:// path
func filesystemFromPath(uri string) (f filesystem.Filesystem, err error) {
	parsed, err := url.Parse(uri)
	if err != nil {
		return
	}

	if parsed.Scheme != "file" {
		return nil, fmt.Errorf("only file scheme is supported")
	}

	path := filepath.FromSlash(uri[7:])

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("path does not exist: %v", path)
	}
	return filesystem.NewOsFilesystem(path), nil
}

// runtimes returns runtimes defined in this repository's filesystem.
// The views are denormalized, using the parent repository's values
// for inherited fields BuildConfig and HealthEndpoints as the default values
// for the runtimes and templates.  The runtimes and templates themselves can
// override these values by specifying new values in thir config files.
func runtimes(fs filesystem.Filesystem, repoCfg repoConfig) (runtimes []Runtime, err error) {
	// Validate templates path
	if err = checkDir(fs, repoCfg.TemplatesPath); err != nil {
		err = fmt.Errorf("templates path '%v' does not exist in repository. %v",
			repoCfg.TemplatesPath, err)
		return
	}

	// For each directory at the path, load it as a runtime
	fis, err := fs.ReadDir(repoCfg.TemplatesPath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue // ignore files and hidden directories
		}

		if fi.Name() == "certs" {
			continue // ignore reserved word "certs"
		}

		runtime := Runtime{
			Name: fi.Name(),
		}

		// Load the runtimeConfig (manifest.yaml) with values from the
		// shared repoCfg as defaults.
		runtime.config, err = loadRuntimeConfig(fs, repoCfg, runtime.Name)
		if err != nil {
			return
		}

		runtime.Templates, err = templates(fs, repoCfg, runtime.config, runtime.Name)
		if err != nil {
			return
		}
		runtimes = append(runtimes, runtime)
	}
	return
}

// templates returns templates currently defined in the given runtime's
// filesystem.  The view is denormalized, using the inherited fields from the
// runtime for defaults of BuildConfig andHealthEndpoints.  The template itself
// can override these by including a manifest.
// The reserved word "scaffolding" is used for repository-defined scaffolding
// code and is not listed as a template.
func templates(fs filesystem.Filesystem, repoCfg repoConfig, runtimeCfg runtimeConfig, runtimeName string) (templates []Template, err error) {
	// Validate runtime path
	runtimePath := path.Join(repoCfg.TemplatesPath, runtimeName)
	if err = checkDir(fs, runtimePath); err != nil {
		err = fmt.Errorf("runtime path '%v' not found. %v", runtimePath, err)
		return
	}

	// Read the directory at the path, load it as a template
	fis, err := fs.ReadDir(runtimePath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue // ignore files and hidden dirs
		}

		if fi.Name() == "scaffolding" {
			continue // ignore the reserved word "scaffolding"
		}

		t := template{
			name:       fi.Name(),
			repository: repoCfg.Name,
			runtime:    runtimeName,
			fs:         filesystem.NewSubFS(path.Join(runtimePath, fi.Name()), fs),
		}

		// update repoCfg with template's manifest.yaml valuse
		t.config, err = loadTemplateConfig(fs, repoCfg, runtimeCfg, runtimeName, t.name)
		if err != nil {
			return
		}

		templates = append(templates, t)
	}
	return
}

// repositoryName returns the given name, which if empty falls back to
// deriving a name from the URI.
func repositoryName(name, uri string) (string, error) {
	// explicit name takes precedence
	if name != "" {
		return name, nil
	}
	// URI-derived is second precedence
	if uri != "" {
		parsed, err := url.Parse(uri)
		if err != nil {
			return "", err
		}
		ss := strings.Split(parsed.Path, "/")
		if len(ss) > 0 {
			// name is the last token with optional '.git' suffix removed
			return strings.TrimSuffix(ss[len(ss)-1], ".git"), nil
		}
	}
	// static default
	return DefaultRepositoryName, nil
}

// loadRepoConfig from the root of the repository's filesystem if it
// exists.  Returned is the repository with any values from the manifest
// set to those of the manifest.
func loadRepoConfig(fs filesystem.Filesystem, repoCfg repoConfig) (repoConfig, error) {
	file, err := fs.Open(manifestFile)
	if err != nil {
		if os.IsNotExist(err) {
			err = nil
		}
		return repoCfg, err
	}
	defer file.Close()

	if err = yaml.NewDecoder(file).Decode(&repoCfg); err != nil {
		return repoCfg, err
	}

	// Default TemplatesPath to CWD
	if repoCfg.TemplatesPath == "" {
		repoCfg.TemplatesPath = "."
	}

	return repoCfg, nil
}

// loadRuntimeConfig from the directory specified (runtime root).  Returned
// is the runtime with values from the manifest populated preferentially.  An
// error is not returned for a missing manifest file (the passed runtime is
// returned), but errors decoding the file are.
func loadRuntimeConfig(fs filesystem.Filesystem, repoCfg repoConfig, runtime string) (runtimeCfg runtimeConfig, err error) {
	// The runtimeConfig is defaulted to the values from the parent (repo)
	runtimeCfg = repoCfg.runtimeConfig // Defaults from the repoCfg

	// If there is a manifest.yaml at the repo level, it can overwrite
	file, err := fs.Open(path.Join(repoCfg.TemplatesPath, runtime, manifestFile))
	if err != nil {
		if os.IsNotExist(err) {
			err = nil
		}
		return // errors other than "Not found" are legitimate
	}
	defer file.Close()
	err = yaml.NewDecoder(file).Decode(&runtimeCfg)
	return
}

// loadTemplateConfig from the directory specified (template root).  Returned
// is the template with values from the manifest populated preferentailly.  An
// error is not returned for a missing manifest file (the passed template is
// returned), but errors decoding the file are.
func loadTemplateConfig(fs filesystem.Filesystem, repoCfg repoConfig, runtimeCfg runtimeConfig, runtimeName, templateName string) (tplCfg templateConfig, err error) {
	// The templateConfig is defaulted to the values from the parent (repo)
	tplCfg = runtimeCfg.templateConfig

	// If there is a manifest.yaml at the template level, it can overwrite
	file, err := fs.Open(path.Join(repoCfg.TemplatesPath, runtimeName, templateName, manifestFile))
	if err != nil {
		if os.IsNotExist(err) {
			err = nil
		}
		return
	}
	defer file.Close()
	err = yaml.NewDecoder(file).Decode(&tplCfg)
	return
}

// check that the given path is an accessible directory or error.
// this checks within the given filesystem, which may have its own root.
func checkDir(fs filesystem.Filesystem, path string) error {
	fi, err := fs.Stat(path)
	if err != nil && os.IsNotExist(err) {
		err = fmt.Errorf("path '%v' not found", path)
	} else if err == nil && !fi.IsDir() {
		err = fmt.Errorf("path '%v' is not a directory", path)
	}
	return err
}

func getGitCloneOptions(uri string) *git.CloneOptions {
	branch := ""
	splitUri := strings.Split(uri, "#")
	if len(splitUri) > 1 {
		uri = splitUri[0]
		branch = splitUri[1]
	}

	opt := &git.CloneOptions{URL: uri, Depth: 1, Tags: git.NoTags,
		RecurseSubmodules: git.NoRecurseSubmodules}
	if branch != "" {
		opt.ReferenceName = plumbing.NewBranchReferenceName(branch)
	}
	return opt
}

// Template from repo for given runtime.
func (r *Repository) Template(runtimeName, name string) (t Template, err error) {
	runtime, err := r.Runtime(runtimeName)
	if err != nil {
		return
	}
	for _, t := range runtime.Templates {
		if t.Name() == name {
			return t, nil
		}
	}
	return nil, ErrTemplateNotFound
}

// Templates returns the set of all templates for a given runtime.
// If runtime not found, an empty list is returned.
func (r *Repository) Templates(runtimeName string) ([]Template, error) {
	for _, runtime := range r.Runtimes {
		if runtime.Name == runtimeName {
			return runtime.Templates, nil
		}
	}
	return nil, nil
}

// Runtime of the given name within the repository.
func (r *Repository) Runtime(name string) (runtime Runtime, err error) {
	if name == "" {
		return Runtime{}, ErrRuntimeRequired
	}
	for _, runtime = range r.Runtimes {
		if runtime.Name == name {
			return runtime, err
		}
	}
	return Runtime{}, ErrRuntimeNotFound
}

// Write all files in the repository to the given path.
func (r *Repository) Write(dest string) (err error) {
	if r.fs == nil {
		return errors.New("the write operation is not supported on this repo")
	}

	fs := r.fs // The FS to copy

	// NOTE
	// We re-load in-memory git repos via a temp directory to avoid what
	// appears to be a missing .git directory in the default worktree FS.
	//
	// This missing .git dir is usually not an issue when utilizing the
	// repository's filesystem (for writing templates, etc), but it does cause
	// problems here where we are writing the entire repository to disk (cloning).
	// We effectively want a full clone with a working tree. So here we do a
	// plain clone first to a temp directory and then copy the files on disk
	// using a regular file copy operation which thus includes the repo metadata.
	if _, ok := r.fs.(filesystem.BillyFilesystem); ok {
		var (
			tempDir string
			clone   *git.Repository
			wt      *git.Worktree
		)
		if tempDir, err = os.MkdirTemp("", "func"); err != nil {
			return
		}
		if clone, err = git.PlainClone(tempDir, false, // not bare
			getGitCloneOptions(r.uri)); err != nil {
			return fmt.Errorf("failed to plain clone repository: %w", err)
		}
		if wt, err = clone.Worktree(); err != nil {
			return fmt.Errorf("failed to get worktree: %w", err)
		}
		fs = filesystem.NewBillyFilesystem(wt.Filesystem)
	}
	return filesystem.CopyFromFS(".", dest, fs)
}

// URL attempts to read the remote git origin URL of the repository.  Best
// effort; returns empty string if the repository is not a git repo or the repo
// has been mutated beyond recognition on disk (ex: removing the origin remote)
func (r *Repository) URL() string {
	uri := r.uri

	// The default builtin repository is indicated by an empty URI.
	// It has no remote URL, and without this check the current working directory
	// would be checked.
	if uri == "" {
		return ""
	}

	// git.PlainOpen does not seem to
	if strings.HasPrefix(uri, "file://") {
		uri = filepath.FromSlash(r.uri[7:])
	}

	repo, err := git.PlainOpen(uri)
	if err != nil {
		return "" // not a git repository
	}

	c, err := repo.Config()
	if err != nil {
		return "" // Has no .git/config or other error.
	}

	ref, _ := repo.Head()
	if _, ok := c.Remotes["origin"]; ok {
		urls := c.Remotes["origin"].URLs
		if len(urls) > 0 {
			return urls[0] + "#" + ref.Name().Short()
		}
	}
	return ""
}