func/repository.go

package function

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/storage/memory"
	"gopkg.in/yaml.v2"
)

const (
	repositoryManifest = "manifest.yaml"
	runtimeManifest    = "manifest.yaml"
	templateManifest   = "manifest.yaml"
)

const (
	// DefaultReadinessEndpoint for final deployed Function instances
	DefaultReadinessEndpoint = "/health/readiness"
	// DefaultLivenessEndpoint for final deployed Function instances
	DefaultLivenessEndpoint = "/health/liveness"
	// DefaultTemplatesPath is the root of the defined repository
	DefaultTemplatesPath = "."
	// DefaultInvocationFormat is a named invocation hint for the convenience
	// helper .Invoke.  It is usually set at the template level.  The default
	// ('http') is a plain HTTP POST.
	DefaultInvocationFormat = "http"

	// Defaults for Builder and Builders not expressly defined as a purposeful
	// delegation of choice.
)

// Repository represents a collection of runtimes, each containing templates.
type Repository struct {
	// Name of the repository.  Naming things and placing them in a hierarchy is
	// the responsibility of the filesystem; metadata the responsibility of the
	// files within this structure. Therefore the name is not part of the repo.
	// This is the same reason a git repository has its name nowhere in .git and
	// does not need a manifest of its contents:  the filesystem itself maintains
	// this information.  This name is the denormalized view of the filesystem,
	// which defines the name as the directory name, and supports being defaulted
	// to the value in the .yaml on initial add, which is stored as DefaultName.
	Name string `yaml:"-"` // use filesystem for names
	// DefaultName is the name indicated by the repository author.
	// Stored in the yaml attribute "name", it is only consulted during initial
	// addition of the repo as the default option.
	DefaultName string `yaml:"name,omitempty"`
	// Version of the repository.
	Version string `yaml:"version,omitempty"`
	// TemplatesPath defines an optional path within the repository at which
	// templates are stored.  By default this is the repository root.
	TemplatesPath string `yaml:"templates,omitempty"`
	// BuildConfig defines builders and buildpacks.  Here it serves as the default
	// option which may be overridden per runtime or per template.
	BuildConfig `yaml:",inline"`
	// HealthEndpoints for all templates in the repository.  Serves as the
	// default option which may be overridden per runtime and per template.
	HealthEndpoints `yaml:"healthEndpoints,omitempty"`
	// BuildEnvs define environment variables for builders that can be used
	//  to parameterize different builders
	BuildEnvs []Env `yaml:"buildEnvs,omitempty"`
	// Runtimes containing Templates loaded from the repo
	Runtimes []Runtime
	// FS is the filesystem underlying the repository, loaded from URI
	// TODO upgrade to fs.FS introduced in go1.16
	FS Filesystem

	// Invocation hints for all templates in this repo
	// (it is more likely this will be set only at the template level)
	Invocation Invocation `yaml:"invocation,omitempty"`

	uri string // URI which was used when initially creating

}

// Runtime is a division of templates within a repository of templates for a
// given runtime (source language plus environmentally available services
// and libraries)
type Runtime struct {
	// Name of the runtime
	Name string `yaml:"-"` // use filesystem for names

	// HealthEndpoints for all templates in the runtime.  May be overridden
	// per template.
	HealthEndpoints `yaml:"healthEndpoints,omitempty"`

	// BuildEnvs for all the templates in the runtime. May be overridden
	// per template.
	BuildEnvs []Env `yaml:"buildEnvs,omitempty"`

	// BuildConfig defines attributes 'builders' and 'buildpacks'.  Here it serves
	// as the default option which may be overridden per template. Note that
	// unlike HealthEndpoints, it is inline, so no 'buildConfig' attribute is
	// added/expected; rather the Buildpacks and Builders are direct descendants
	// of Runtime.
	BuildConfig `yaml:",inline"`

	// Invocation hints for all templates in this runtime
	// (it is more likely this will be set only at the template level)
	Invocation Invocation `yaml:"invocation,omitempty"`

	// Templates defined for the runtime
	Templates []Template
}

// HealthEndpoints specify the liveness and readiness endpoints for a Runtime
type HealthEndpoints struct {
	Liveness  string `yaml:"liveness,omitempty"`
	Readiness string `yaml:"readiness,omitempty"`
}

// BuildConfig defines builders and buildpacks
type BuildConfig struct {
	Buildpacks []string          `yaml:"buildpacks,omitempty"`
	Builders   map[string]string `yaml:"builders,omitempty"`
}

// NewRepository creates a repository instance from any of: a path on disk, a
// remote or local URI, or from the embedded default repo if uri not provided.
// Name (optional), if provided takes precedence over name derived from repo at
//   the given URI.
// URI (optional), the path either locally or remote from which to load
//    the repository files.  If not provided, the internal default is assumed.
func NewRepository(name, uri string) (r Repository, err error) {
	r = Repository{
		uri: uri,
		HealthEndpoints: HealthEndpoints{
			Liveness:  DefaultLivenessEndpoint,
			Readiness: DefaultLivenessEndpoint,
		},
		Invocation: Invocation{Format: DefaultInvocationFormat},
	}
	r.FS, err = filesystemFromURI(uri) // Get a Filesystem from the URI
	if err != nil {
		return Repository{}, fmt.Errorf("failed to get repository from URI (%q): %w", uri, err)
	}
	r, err = applyRepositoryManifest(r) // apply optional manifest to r
	if err != nil {
		return
	}
	r.Name, err = repositoryDefaultName(r.DefaultName, uri) // choose default name
	if err != nil {
		return
	}
	if name != "" { // If provided, the explicit name takes precidence
		r.Name = name
	}
	r.Runtimes, err = repositoryRuntimes(r) // load templates grouped by runtime
	return
}

// filesystemFromURI returns a filesystem from the data located at the
// given URI.  If URI is not provided, indicates the embedded repo should
// be loaded.  URI can be a remote git repository (http:// https:// etc.),
// or a local file path (file://) which can be a git repo or a plain directory.
func filesystemFromURI(uri string) (f Filesystem, err error) {
	// If not provided, indicates embedded.
	if uri == "" {
		return EmbeddedTemplatesFS, nil
	}

	// Attempt to get a filesystem from the uri as a remote repo.
	f, err = filesystemFromRepo(uri)
	if f != nil || err != nil {
		return // found a filesystem and/or an error
	}

	// Attempt to get a filesystem from the uri as a file path.
	return filesystemFromPath(uri)
}

// filesystemFromRepo attempts to fetch a filesystem from a git repository
// indicated by the given URI.  Returns nil if there is not a repo at the URI.
func filesystemFromRepo(uri string) (Filesystem, error) {
	clone, err := git.Clone(
		memory.NewStorage(),
		memfs.New(),
		&git.CloneOptions{URL: uri, Depth: 1, Tags: git.NoTags,
			RecurseSubmodules: git.NoRecurseSubmodules,
		})
	if err != nil {
		if isRepoNotFoundError(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("failed to clone repository: %w", err)
	}
	wt, err := clone.Worktree()
	if err != nil {
		return nil, err
	}
	return billyFilesystem{fs: wt.Filesystem}, nil
}

// isRepoNotFoundError returns true if the error is a
// "repository not found" error.
func isRepoNotFoundError(err error) bool {
	// This would be better if the error being tested for was typed, but it is
	// currently a simple string value comparison.
	return (err != nil && err.Error() == "repository not found")
}

// filesystemFromPath attempts to return a filesystem from a URI as a file:// path
func filesystemFromPath(uri string) (f Filesystem, err error) {
	parsed, err := url.Parse(uri)
	if err != nil {
		return
	}

	if parsed.Scheme != "file" {
		return nil, fmt.Errorf("only file scheme is supported")
	}

	path := filepath.FromSlash(uri[7:])

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("path does not exist: %v", path)
	}
	return osFilesystem{root: path}, nil
}

// repositoryRuntimes returns runtimes defined in this repository's filesystem.
// The views are denormalized, using the parent repository's values
// for inherited fields BuildConfig and HealthEndpoints as the default values
// for the runtimes and templates.  The runtimes and templates themselves can
// override these values by specifying new values in thir config files.
func repositoryRuntimes(r Repository) (runtimes []Runtime, err error) {
	runtimes = []Runtime{}

	// Validate custom path if defined
	if r.TemplatesPath != "" {
		if err = checkDir(r.FS, r.TemplatesPath); err != nil {
			err = fmt.Errorf("templates path '%v' does not exist in repo '%v'. %v",
				r.TemplatesPath, r.Name, err)
			return
		}
	}

	// Load runtimes
	if r.TemplatesPath == "" {
		r.TemplatesPath = "."
	}

	fis, err := r.FS.ReadDir(r.TemplatesPath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		// ignore files and hidden dirs
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		// Runtime, defaulted to values inherited from the repository
		runtime := Runtime{
			Name:            fi.Name(),
			BuildConfig:     r.BuildConfig,
			HealthEndpoints: r.HealthEndpoints,
			BuildEnvs:       r.BuildEnvs,
			Invocation:      r.Invocation,
		}
		// Runtime Manifest
		// Load the file if it exists, which may override values inherited from the
		// repo such as builders, buildpacks and health endpoints.
		runtime, err = applyRuntimeManifest(r, runtime)
		if err != nil {
			return
		}

		// Runtime Templates
		// Load from repo filesystem for runtime. Will inherit values from the
		// runtime such as BuildConfig, HealthEndpoints etc.
		runtime.Templates, err = runtimeTemplates(r, runtime)
		if err != nil {
			return
		}
		runtimes = append(runtimes, runtime)
	}
	return
}

// runtimeTemplates returns templates currently defined in the given runtime's
// filesystem.  The view is denormalized, using the inherited fields from the
// runtime for defaults of BuildConfig andHealthEndpoints.  The template itself
// can override these by including a manifest.
func runtimeTemplates(r Repository, runtime Runtime) (templates []Template, err error) {
	templates = []Template{}

	// Validate runtime directory exists and is a directory
	runtimePath := path.Join(r.TemplatesPath, runtime.Name)
	if err = checkDir(r.FS, runtimePath); err != nil {
		err = fmt.Errorf("runtime path '%v' not found. %v", runtimePath, err)
		return
	}

	// Read the directory, loading each template.
	fis, err := r.FS.ReadDir(runtimePath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		// ignore files and hidden dirs
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		// Template, defaulted to values inherited from the runtime
		t := Template{
			Name:            fi.Name(),
			Repository:      r.Name,
			Runtime:         runtime.Name,
			BuildConfig:     runtime.BuildConfig,
			HealthEndpoints: runtime.HealthEndpoints,
			BuildEnvs:       runtime.BuildEnvs,
			Invocation:      runtime.Invocation,
		}

		// Template Manifeset
		// Load manifest file if it exists, which may override values inherited from
		// the runtime/repo.
		t, err = applyTemplateManifest(r, t)
		if err != nil {
			return
		}
		templates = append(templates, t)
	}
	return
}

// repositoryDefaultName returns the given name, which if empty falls back to
// deriving a name from the URI, which if empty then falls back to the
// statically defined default DefaultRepositoryName.
func repositoryDefaultName(name, uri string) (string, error) {
	// explicit name takes precidence
	if name != "" {
		return name, nil
	}
	// URI-derived is second precidence
	if uri != "" {
		parsed, err := url.Parse(uri)
		if err != nil {
			return "", err
		}
		ss := strings.Split(parsed.Path, "/")
		if len(ss) > 0 {
			// name is the last token with optional '.git' suffix removed
			return strings.TrimSuffix(ss[len(ss)-1], ".git"), nil
		}
	}
	// static default
	return DefaultRepositoryName, nil
}

// applyRepositoryManifest from the root of the repository's filesystem if it
// exists.  Returned is the repository with any values from the manifest
// set to those of the manifest.
func applyRepositoryManifest(r Repository) (Repository, error) {
	file, err := r.FS.Open(repositoryManifest)
	if err != nil {
		if os.IsNotExist(err) {
			return r, nil
		}
		return r, err
	}
	decoder := yaml.NewDecoder(file)
	return r, decoder.Decode(&r)
}

// applyRuntimeManifest from the directory specified (runtime root).  Returned
// is the runtime with values from the manifest populated preferentially.  An
// error is not returned for a missing manifest file (the passed runtime is
// returned), but errors decoding the file are.
func applyRuntimeManifest(repo Repository, runtime Runtime) (Runtime, error) {
	file, err := repo.FS.Open(path.Join(repo.TemplatesPath, runtime.Name, runtimeManifest))
	if err != nil {
		if os.IsNotExist(err) {
			return runtime, nil
		}
		return runtime, err
	}
	decoder := yaml.NewDecoder(file)
	return runtime, decoder.Decode(&runtime)
}

// applyTemplateManifest from the directory specified (template root).  Returned
// is the template with values from the manifest populated preferentailly.  An
// error is not returned for a missing manifest file (the passed template is
// returned), but errors decoding the file are.
func applyTemplateManifest(repo Repository, t Template) (Template, error) {
	file, err := repo.FS.Open(path.Join(repo.TemplatesPath, t.Runtime, t.Name, templateManifest))
	if err != nil {
		if os.IsNotExist(err) {
			return t, nil
		}
		return t, err
	}
	decoder := yaml.NewDecoder(file)
	return t, decoder.Decode(&t)
}

// check that the given path is an accessible directory or error.
// this checks within the given filesystem, which may have its own root.
func checkDir(fs Filesystem, path string) error {
	fi, err := fs.Stat(path)
	if err != nil && os.IsNotExist(err) {
		err = fmt.Errorf("path '%v' not found", path)
	} else if err == nil && !fi.IsDir() {
		err = fmt.Errorf("path '%v' is not a directory", path)
	}
	return err
}

// Template from repo for given runtime.
func (r *Repository) Template(runtimeName, name string) (t Template, err error) {
	runtime, err := r.Runtime(runtimeName)
	if err != nil {
		return
	}
	for _, t := range runtime.Templates {
		if t.Name == name {
			return t, nil
		}
	}
	return Template{}, ErrTemplateNotFound
}

// Templates returns the set of all templates for a given runtime.
// If runtime not found, an empty list is returned.
func (r *Repository) Templates(runtimeName string) ([]Template, error) {
	for _, runtime := range r.Runtimes {
		if runtime.Name == runtimeName {
			return runtime.Templates, nil
		}
	}
	return []Template{}, nil
}

// Runtime of the given name within the repository.
func (r *Repository) Runtime(name string) (runtime Runtime, err error) {
	for _, runtime = range r.Runtimes {
		if runtime.Name == name {
			return runtime, err
		}
	}
	return Runtime{}, ErrRuntimeNotFound
}

// Write all files in the repository to the given path.
func (r *Repository) Write(path string) (err error) {
	fs := r.FS // The FS to copy

	// NOTE
	// We re-load in-memory git repos via a temp directory to avoid what
	// appears to be a missing .git directory in the default worktree FS.
	//
	// This missing .git dir is usually not an issue when utilizing the
	// repository's filesystem (for writing templates, etc), but it does cause
	// problems here where we are writing the entire repository to disk (cloning).
	// We effectively want a full clone with a working tree. So here we do a
	// plain clone first to a temp directory and then copy the files on disk
	// using a regular file copy operation which thus includes the repo metadata.
	if _, ok := r.FS.(billyFilesystem); ok {
		var (
			tempDir string
			clone   *git.Repository
			wt      *git.Worktree
		)
		if tempDir, err = ioutil.TempDir("", "func"); err != nil {
			return
		}
		if clone, err = git.PlainClone(tempDir, false, // not bare
			&git.CloneOptions{URL: r.uri, Depth: 1, Tags: git.NoTags,
				RecurseSubmodules: git.NoRecurseSubmodules}); err != nil {
			return fmt.Errorf("failed to plain clone repository: %w", err)
		}
		if wt, err = clone.Worktree(); err != nil {
			return fmt.Errorf("failed to get worktree: %w", err)
		}
		fs = billyFilesystem{fs: wt.Filesystem}
	}
	return copy(".", path, fs)
}

// URL attempts to read the remote git origin URL of the repository.  Best
// effort; returns empty string if the repository is not a git repo or the repo
// has been mutated beyond recognition on disk (ex: removing the origin remote)
func (r *Repository) URL() string {
	uri := r.uri

	// The default builtin repository is indicated by an empty URI.
	// It has no remote URL, and without this check the current working directory
	// would be checked.
	if uri == "" {
		return ""
	}

	// git.PlainOpen does not seem to
	if strings.HasPrefix(uri, "file://") {
		uri = filepath.FromSlash(r.uri[7:])
	}

	repo, err := git.PlainOpen(uri)
	if err != nil {
		return "" // not a git repository
	}

	c, err := repo.Config()
	if err != nil {
		return "" // Has no .git/config or other error.
	}

	if _, ok := c.Remotes["origin"]; ok {
		urls := c.Remotes["origin"].URLs
		if len(urls) > 0 {
			return urls[0]
		}
	}
	return ""
}