func/repository.go

package function

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/storage/memory"
	"gopkg.in/yaml.v2"
)

const (
	repositoryManifest = "manifest.yaml"
	runtimeManifest    = "manifest.yaml"
	templateManifest   = "manifest.yaml"
)

const (
	// DefaultReadinessEndpoint for final deployed Function instances
	DefaultReadinessEndpoint = "/health/readiness"
	// DefaultLivenessEndpoint for final deployed Function instances
	DefaultLivenessEndpoint = "/health/liveness"
	// DefaultTemplatesPath is the root of the defined repository
	DefaultTemplatesPath = "."

	// Defaults for Builder and Builders not expressly defined as a purposeful
	// delegation of choice.
)

// Repository represents a collection of runtimes, each containing templates.
type Repository struct {
	// Name of the repository.  Naming things and placing them in a hierarchy is
	// the responsibility of the filesystem; metadata the responsibility of the
	// files within this structure. Therefore the name is not part of the repo.
	// This is the same reason a git repository has its name nowhere in .git and
	// does not need a manifest of its contents:  the filesystem itself maintains
	// this information.  This name is the denormalized view of the filesystem,
	// which defines the name as the directory name, and supports being defaulted
	// to the value in the .yaml on initial add, which is stored as DefaultName.
	Name string `yaml:"-"` // use filesystem for names
	// DefaultName is the name indicated by the repository author.
	// Stored in the yaml attribute "name", it is only consulted during initial
	// addition of the repo as the default option.
	DefaultName string `yaml:"name,omitempty"`
	// Version of the repository.
	Version string `yaml:"version,omitempty"`
	// TemplatesPath defines an optional path within the repository at which
	// templates are stored.  By default this is the repository root.
	TemplatesPath string `yaml:"templates,omitempty"`
	// BuildConfig defines builders and buildpacks.  Here it serves as the default
	// option which may be overridden per runtime or per template.
	BuildConfig `yaml:",inline"`
	// HealthEndpoints for all templates in the repository.  Serves as the
	// default option which may be overridden per runtime and per template.
	HealthEndpoints `yaml:"healthEndpoints,omitempty"`
	// BuildEnvs define environment variables for builders that can be used
	//  to parameterize different builders
	BuildEnvs []Env `yaml:"buildEnvs,omitempty"`
	// Runtimes containing Templates loaded from the repo
	Runtimes []Runtime
	// FS is the filesystem underlying the repository, loaded from URI
	// TODO upgrade to fs.FS introduced in go1.16
	FS Filesystem

	uri string // URI which was used when initially creating

}

// Runtime is a division of templates within a repository of templates for a
// given runtime (source language plus environmentally available services
// and libraries)
type Runtime struct {
	// Name of the runtime
	Name string `yaml:"-"` // use filesystem for names

	// HealthEndpoints for all templates in the runtime.  May be overridden
	// per template.
	HealthEndpoints `yaml:"healthEndpoints,omitempty"`

	// BuildEnvs for all the templates in the runtime. May be overridden
	// per template.
	BuildEnvs []Env `yaml:"buildEnvs,omitempty"`

	// BuildConfig defines attributes 'builders' and 'buildpacks'.  Here it serves
	// as the default option which may be overridden per template. Note that
	// unlike HealthEndpoints, it is inline, so no 'buildConfig' attribute is
	// added/expected; rather the Buildpacks and Builders are direct descendants
	// of Runtime.
	BuildConfig `yaml:",inline"`

	// Templates defined for the runtime
	Templates []Template
}

// HealthEndpoints specify the liveness and readiness endpoints for a Runtime
type HealthEndpoints struct {
	Liveness  string `yaml:"liveness,omitempty"`
	Readiness string `yaml:"readiness,omitempty"`
}

// BuildConfig defines builders and buildpacks
type BuildConfig struct {
	Buildpacks []string          `yaml:"buildpacks,omitempty"`
	Builders   map[string]string `yaml:"builders,omitempty"`
}

// NewRepository creates a repository instance from any of: a path on disk, a
// remote or local URI, or from the embedded default repo if uri not provided.
// Name (optional), if provided takes precedence over name derived from repo at
//   the given URI.
// URI (optional), the path either locally or remote from which to load
//    the repository files.  If not provided, the internal default is assumed.
func NewRepository(name, uri string) (r Repository, err error) {
	r = Repository{
		uri: uri,
		HealthEndpoints: HealthEndpoints{
			Liveness:  DefaultLivenessEndpoint,
			Readiness: DefaultLivenessEndpoint,
		},
	}
	r.FS, err = filesystemFromURI(uri) // Get a Filesystem from the URI
	if err != nil {
		return
	}
	r, err = applyRepositoryManifest(r) // apply optional manifest to r
	if err != nil {
		return
	}
	r.Name, err = repositoryDefaultName(r.DefaultName, uri) // choose default name
	if err != nil {
		return
	}
	if name != "" { // If provided, the explicit name takes precidence
		r.Name = name
	}
	r.Runtimes, err = repositoryRuntimes(r) // load templates grouped by runtime
	return
}

// filesystemFromURI returns a filesystem from the data located at the
// given URI.  If URI is not provided, indicates the embedded repo should
// be loaded.  URI can be a remote git repository (http:// https:// etc.),
// or a local file path (file://) which can be a git repo or a plain directory.
func filesystemFromURI(uri string) (f Filesystem, err error) {
	// If not provided, indicates embedded.
	if uri == "" {
		return pkgerFilesystem{}, nil
	}

	// Attempt to get a filesystem from the uri as a remote repo.
	f, err = filesystemFromRepo(uri)
	if f != nil || err != nil {
		return // found a filesystem and/or an error
	}

	// Attempt to get a filesystem from the uri as a file path.
	return filesystemFromPath(uri)
}

// filesystemFromRepo attempts to fetch a filesystem from a git repository
// indicated by the given URI.  Returns nil if there is not a repo at the URI.
func filesystemFromRepo(uri string) (Filesystem, error) {
	clone, err := git.Clone(
		memory.NewStorage(),
		memfs.New(),
		&git.CloneOptions{URL: uri, Depth: 1, Tags: git.NoTags,
			RecurseSubmodules: git.NoRecurseSubmodules,
		})
	if err != nil {
		if isRepoNotFoundError(err) {
			err = nil // no repo at location is an expected condition
		}
		return nil, err
	}
	wt, err := clone.Worktree()
	if err != nil {
		return nil, err
	}
	return billyFilesystem{fs: wt.Filesystem}, nil
}

// isRepoNotFoundError returns true if the error is a
// "repository not found" error.
func isRepoNotFoundError(err error) bool {
	// This would be better if the error being tested for was typed, but it is
	// currently a simple string value comparison.
	return (err != nil && err.Error() == "repository not found")
}

// filesystemFromPath attempts to return a filesystem from a URI as a file:// path
func filesystemFromPath(uri string) (f Filesystem, err error) {
	parsed, err := url.Parse(uri)
	if err != nil {
		return
	}
	if _, err := os.Stat(parsed.Path); os.IsNotExist(err) {
		return nil, fmt.Errorf("path does not exist: %v", parsed.Path)
	}
	return osFilesystem{root: parsed.Path}, nil
}

// repositoryRuntimes returns runtimes defined in this repository's filesystem.
// The views are denormalized, using the parent repository's values
// for inherited fields BuildConfig and HealthEndpoints as the default values
// for the runtimes and templates.  The runtimes and templates themselves can
// override these values by specifying new values in thir config files.
func repositoryRuntimes(r Repository) (runtimes []Runtime, err error) {
	runtimes = []Runtime{}

	// Validate custom path if defined
	if r.TemplatesPath != "" {
		if err = checkDir(r.FS, r.TemplatesPath); err != nil {
			err = fmt.Errorf("templates path '%v' does not exist in repo '%v'. %v",
				r.TemplatesPath, r.Name, err)
			return
		}
	}

	// Load runtimes
	if r.TemplatesPath == "" {
		r.TemplatesPath = "/"
	}

	fis, err := r.FS.ReadDir(r.TemplatesPath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		// ignore files and hidden dirs
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		// Runtime, defaulted to values inherited from the repository
		runtime := Runtime{
			Name:            fi.Name(),
			BuildConfig:     r.BuildConfig,
			HealthEndpoints: r.HealthEndpoints,
			BuildEnvs:       r.BuildEnvs,
		}
		// Runtime Manifest
		// Load the file if it exists, which may override values inherited from the
		// repo such as builders, buildpacks and health endpoints.
		runtime, err = applyRuntimeManifest(r, runtime)
		if err != nil {
			return
		}

		// Runtime Templates
		// Load from repo filesystem for runtime. Will inherit values from the
		// runtime such as BuildConfig, HealthEndpoints etc.
		runtime.Templates, err = runtimeTemplates(r, runtime)
		if err != nil {
			return
		}
		runtimes = append(runtimes, runtime)
	}
	return
}

// runtimeTemplates returns templates currently defined in the given runtime's
// filesystem.  The view is denormalized, using the inherited fields from the
// runtime for defaults of BuildConfig andHealthEndpoints.  The template itself
// can override these by including a manifest.
func runtimeTemplates(r Repository, runtime Runtime) (templates []Template, err error) {
	templates = []Template{}

	// Validate runtime directory exists and is a directory
	runtimePath := filepath.Join(r.TemplatesPath, runtime.Name)
	if err = checkDir(r.FS, runtimePath); err != nil {
		err = fmt.Errorf("runtime path '%v' not found. %v", runtimePath, err)
		return
	}

	// Read the directory, loading each template.
	fis, err := r.FS.ReadDir(runtimePath)
	if err != nil {
		return
	}
	for _, fi := range fis {
		// ignore files and hidden dirs
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		// Template, defaulted to values inherited from the runtime
		t := Template{
			Name:            fi.Name(),
			Repository:      r.Name,
			Runtime:         runtime.Name,
			BuildConfig:     runtime.BuildConfig,
			HealthEndpoints: runtime.HealthEndpoints,
			BuildEnvs:       runtime.BuildEnvs,
		}

		// Template Manifeset
		// Load manifest file if it exists, which may override values inherited from
		// the runtime/repo.
		t, err = applyTemplateManifest(r, t)
		if err != nil {
			return
		}
		templates = append(templates, t)
	}
	return
}

// repositoryDefaultName returns the given name, which if empty falls back to
// deriving a name from the URI, which if empty then falls back to the
// statically defined default DefaultRepositoryName.
func repositoryDefaultName(name, uri string) (string, error) {
	// explicit name takes precidence
	if name != "" {
		return name, nil
	}
	// URI-derived is second precidence
	if uri != "" {
		parsed, err := url.Parse(uri)
		if err != nil {
			return "", err
		}
		ss := strings.Split(parsed.Path, "/")
		if len(ss) > 0 {
			// name is the last token with optional '.git' suffix removed
			return strings.TrimSuffix(ss[len(ss)-1], ".git"), nil
		}
	}
	// static default
	return DefaultRepositoryName, nil
}

// applyRepositoryManifest from the root of the repository's filesystem if it
// exists.  Returned is the repository with any values from the manifest
// set to those of the manifest.
func applyRepositoryManifest(r Repository) (Repository, error) {
	file, err := r.FS.Open(repositoryManifest)
	if err != nil {
		if os.IsNotExist(err) {
			return r, nil
		}
		return r, err
	}
	decoder := yaml.NewDecoder(file)
	return r, decoder.Decode(&r)
}

// applyRuntimeManifest from the directory specified (runtime root).  Returned
// is the runtime with values from the manifest populated preferentially.  An
// error is not returned for a missing manifest file (the passed runtime is
// returned), but errors decoding the file are.
func applyRuntimeManifest(repo Repository, runtime Runtime) (Runtime, error) {
	file, err := repo.FS.Open(filepath.Join(repo.TemplatesPath, runtime.Name, runtimeManifest))
	if err != nil {
		if os.IsNotExist(err) {
			return runtime, nil
		}
		return runtime, err
	}
	decoder := yaml.NewDecoder(file)
	return runtime, decoder.Decode(&runtime)
}

// applyTemplateManifest from the directory specified (template root).  Returned
// is the template with values from the manifest populated preferentailly.  An
// error is not returned for a missing manifest file (the passed template is
// returned), but errors decoding the file are.
func applyTemplateManifest(repo Repository, t Template) (Template, error) {
	file, err := repo.FS.Open(filepath.Join(repo.TemplatesPath, t.Runtime, t.Name, templateManifest))
	if err != nil {
		if os.IsNotExist(err) {
			return t, nil
		}
		return t, err
	}
	decoder := yaml.NewDecoder(file)
	return t, decoder.Decode(&t)
}

// check that the given path is an accessible directory or error.
// this checks within the given filesystem, which may have its own root.
func checkDir(fs Filesystem, path string) error {
	fi, err := fs.Stat(path)
	if err != nil && os.IsNotExist(err) {
		err = fmt.Errorf("path '%v' not found", path)
	} else if err == nil && !fi.IsDir() {
		err = fmt.Errorf("path '%v' is not a directory", path)
	}
	return err
}

// Template from repo for given runtime.
func (r *Repository) Template(runtimeName, name string) (t Template, err error) {
	runtime, err := r.Runtime(runtimeName)
	if err != nil {
		return
	}
	for _, t := range runtime.Templates {
		if t.Name == name {
			return t, nil
		}
	}
	return Template{}, ErrTemplateNotFound
}

// Templates returns the set of all templates for a given runtime.
// If runtime not found, an empty list is returned.
func (r *Repository) Templates(runtimeName string) ([]Template, error) {
	for _, runtime := range r.Runtimes {
		if runtime.Name == runtimeName {
			return runtime.Templates, nil
		}
	}
	return []Template{}, nil
}

// Runtime of the given name within the repository.
func (r *Repository) Runtime(name string) (runtime Runtime, err error) {
	for _, runtime = range r.Runtimes {
		if runtime.Name == name {
			return runtime, err
		}
	}
	return Runtime{}, ErrRuntimeNotFound
}

// Write all files in the repository to the given path.
func (r *Repository) Write(path string) error {
	// NOTE: Writing internal .git directory does not work
	//
	// A quirk of the git library's implementation is that the filesystem
	// returned does not include the .git directory.  This is usually not an
	// issue when utilizing the repository's filesystem (for writing templates),
	// but it does cause problems here (used for installing a repo locally) where
	// we effectively want a full clone.
	// TODO: switch to using a temp directory?

	return copy("/", path, r.FS) // copy 'all' to 'dest' from 'FS'
}

// URL attempts to read the remote git origin URL of the repository.  Best
// effort; returns empty string if the repository is not a git repo or the repo
// has been mutated beyond recognition on disk (ex: removing the origin remote)
func (r *Repository) URL() string {
	repo, err := git.PlainOpen(r.uri)
	if err != nil {
		return "" // not a git repository
	}

	c, err := repo.Config()
	if err != nil {
		return "" // Has no .git/config or other error.
	}

	if _, ok := c.Remotes["origin"]; ok {
		urls := c.Remotes["origin"].URLs
		if len(urls) > 0 {
			return urls[0]
		}
	}
	return ""
}